class Braingasm::Tokenizer
Attributes
column_number[R]
input[RW]
line_numer[R]
Public Class Methods
new(input)
click to toggle source
Calls superclass method
# File lib/braingasm/tokenizer.rb, line 8 def initialize(input) @line_numer = 1 @column_number = 0 scanner = StringScanner.new(input) super() do |y| loop do line_numer, column_number = @line_numer, @column_number while scanner.skip(/\s/) if scanner.beginning_of_line? line_numer += 1 column_number = 0 else column_number += 1 end end break if scanner.eos? column_number += 1 @line_numer, @column_number = line_numer, column_number y << read_token(scanner) end end end
Private Instance Methods
read_token(scanner)
click to toggle source
# File lib/braingasm/tokenizer.rb, line 37 def read_token(scanner) if scanner.scan(/\d+/) scanner.matched.to_i elsif scanner.scan(/"/) s = scanner.scan(/[^"]*/) scanner.skip(/"/) s else @@simple_tokens[scanner.scan(/\S/)] || :unknown end end