module Tildeath::Lexer
Constants
- TOKENS
token types
Public Class Methods
lex(input)
click to toggle source
# File lib/tildeath/lexer.rb, line 33 def self.lex(input) # tokens found so far tokens = [] # current position in script pos = 0 line_number = 0 column = 0 match = nil # while pos isn't outside the script... while pos < input.length if input[pos] != "\n" column += 1 else line_number += 1 column = 0 end # for each token type... good = TOKENS.any? do |sym, regex| # try to match it at the current position in the script match = input.match(regex, pos) # skip to next token type unless it matched at the current position next unless match && match.begin(0) == pos # ignore whitespace and comments unless [:WS, :COMMENT].include?(sym) # add new token to list of found tokens # if it's an identifier, save the actual text found as well tokens << Token.new(sym, line_number, column, sym == :IDENT ? match[0] : nil) end # move current position to just after the end of the found token pos += match[0].length true end fail TildeathError.new(line_number, column), "error: unrecognized token #{input[pos]}" unless good end return tokens end