class Rgo::Tokenizer
Public Class Methods
new(source)
click to toggle source
# File lib/rgo/tokenizer.rb, line 5 def initialize(source) @source = source end
Public Instance Methods
tokens()
click to toggle source
# File lib/rgo/tokenizer.rb, line 9 def tokens @q = [] scanner = StringScanner.new(@source) until scanner.empty? case when m=scanner.scan(/\s+/) blank_lines = m.count("\n") - 1 blank_lines.times do @q.push [:BLANK_LINE, ""] end when m = scanner.scan(Expression::INTEGER) @q.push [:INTEGER, m.to_i] when m = scanner.scan(Expression::KEYWORDS) @q.push [:"KEYWORD_#{m.upcase}", m] when m = scanner.scan(Expression::COMMENT) @q.push [:COMMENT, m[1..-1]] when m = scanner.scan(Expression::STRING) @q.push [:STRING, m[1..-2]] when m = scanner.scan(Expression::CONSTANT) @q.push [:CONSTANT, m] when m = scanner.scan(Expression::IDENTIFIER) @q.push [:IDENTIFIER, m] when m = scanner.scan(/==/) @q.push [:EQUAL, "=="] when m = scanner.scan(/=/) @q.push [:KEYWORD_ASSIGN, "="] when m = scanner.scan(/\+/) @q.push [:PLUS, "+"] when m = scanner.scan(/\-/) @q.push [:MINUS, "-"] when m = scanner.scan(/\*/) @q.push [:MULTIPLY, "*"] when m = scanner.scan(/\//) @q.push [:DIVIDE, "/"] when m = scanner.scan(/>/) @q.push [:GREATER, ">"] when m = scanner.scan(/</) @q.push [:LESS, "<"] when m = scanner.scan(/\%/) @q.push [:MOD, "%"] when m = scanner.scan(/\(/) @q.push [:LPAREN, "("] when m = scanner.scan(/\)/) @q.push [:RPAREN, ")"] when m = scanner.scan(/\,/) @q.push [:COMMA, ","] when m = scanner.scan(/\./) @q.push [:DOT, "."] when m = scanner.scan(/\:\:/) @q.push [:DCOLON, "::"] when m = scanner.scan(/\:/) @q.push [:COLON, ":"] when m = scanner.scan(Expression::INSTANCE_VARIABLE) @q.push [:INSTANCE_VARIABLE, m[1..-1]] when m = scanner.scan(/.|\n/) @q.push [m, m] end end @q.push [false, false] @q end