class Braingasm::Tokenizer

Attributes

column_number[R]
input[RW]
line_numer[R]

Public Class Methods

new(input) click to toggle source
Calls superclass method
# File lib/braingasm/tokenizer.rb, line 8
def initialize(input)
  @line_numer = 1
  @column_number = 0

  scanner = StringScanner.new(input)

  super() do |y|
    loop do
      line_numer, column_number = @line_numer, @column_number

      while scanner.skip(/\s/)
        if scanner.beginning_of_line?
          line_numer += 1
          column_number = 0
        else
          column_number += 1
        end
      end

      break if scanner.eos?

      column_number += 1
      @line_numer, @column_number = line_numer, column_number
      y << read_token(scanner)
    end
  end
end

Private Instance Methods

read_token(scanner) click to toggle source
# File lib/braingasm/tokenizer.rb, line 37
def read_token(scanner)
  if scanner.scan(/\d+/)
    scanner.matched.to_i
  elsif scanner.scan(/"/)
    s = scanner.scan(/[^"]*/)
    scanner.skip(/"/)
    s
  else
    @@simple_tokens[scanner.scan(/\S/)] || :unknown
  end
end