class Rgo::Tokenizer

Public Class Methods

new(source) click to toggle source
# File lib/rgo/tokenizer.rb, line 5
def initialize(source)
  @source = source
end

Public Instance Methods

tokens() click to toggle source
# File lib/rgo/tokenizer.rb, line 9
def tokens
  @q = []
  scanner = StringScanner.new(@source)

  until scanner.empty?
    case
      when m=scanner.scan(/\s+/)
        blank_lines = m.count("\n") - 1
        blank_lines.times do
          @q.push [:BLANK_LINE, ""]
        end
      when m = scanner.scan(Expression::INTEGER)
        @q.push [:INTEGER, m.to_i]
      when m = scanner.scan(Expression::KEYWORDS)
        @q.push [:"KEYWORD_#{m.upcase}", m]
      when m = scanner.scan(Expression::COMMENT)
        @q.push [:COMMENT, m[1..-1]]
      when m = scanner.scan(Expression::STRING)
        @q.push [:STRING, m[1..-2]]
      when m = scanner.scan(Expression::CONSTANT)
        @q.push [:CONSTANT, m]
      when m = scanner.scan(Expression::IDENTIFIER)
        @q.push [:IDENTIFIER, m]
      when m = scanner.scan(/==/)
        @q.push [:EQUAL, "=="]
      when m = scanner.scan(/=/)
        @q.push [:KEYWORD_ASSIGN, "="]
      when m = scanner.scan(/\+/)
        @q.push [:PLUS, "+"]
      when m = scanner.scan(/\-/)
        @q.push [:MINUS, "-"]
      when m = scanner.scan(/\*/)
        @q.push [:MULTIPLY, "*"]
      when m = scanner.scan(/\//)
        @q.push [:DIVIDE, "/"]
      when m = scanner.scan(/>/)
        @q.push [:GREATER, ">"]
      when m = scanner.scan(/</)
        @q.push [:LESS, "<"]
      when m = scanner.scan(/\%/)
        @q.push [:MOD, "%"]
      when m = scanner.scan(/\(/)
        @q.push [:LPAREN, "("]
      when m = scanner.scan(/\)/)
        @q.push [:RPAREN, ")"]
      when m = scanner.scan(/\,/)
        @q.push [:COMMA, ","]
      when m = scanner.scan(/\./)
        @q.push [:DOT, "."]
      when m = scanner.scan(/\:\:/)
        @q.push [:DCOLON, "::"]
      when m = scanner.scan(/\:/)
        @q.push [:COLON, ":"]
      when m = scanner.scan(Expression::INSTANCE_VARIABLE)
        @q.push [:INSTANCE_VARIABLE, m[1..-1]]
      when m = scanner.scan(/.|\n/)
        @q.push [m, m]
    end
  end

  @q.push [false, false]

  @q
end