class Keisan::Tokenizer
Constants
- TOKEN_CLASSES
- TOKEN_REGEX
Attributes
expression[R]
tokens[R]
Public Class Methods
new(expression)
click to toggle source
# File lib/keisan/tokenizer.rb, line 26 def initialize(expression) @expression = expression portions = StringAndGroupParser.new(expression).portions.reject do |portion| portion.is_a? StringAndGroupParser::CommentPortion end @tokens = portions.inject([]) do |tokens, portion| case portion when StringAndGroupParser::StringPortion tokens << Tokens::String.new(portion.escaped_string) when StringAndGroupParser::GroupPortion tokens << Tokens::Group.new(portion.to_s) when StringAndGroupParser::OtherPortion scan = portion.to_s.scan(TOKEN_REGEX) tokens += tokenize!(scan) end tokens end end
Private Instance Methods
tokenize!(scan)
click to toggle source
# File lib/keisan/tokenizer.rb, line 50 def tokenize!(scan) scan.map do |scan_result| i = scan_result.find_index {|token| !token.nil?} token_string = scan_result[i] token = TOKEN_CLASSES[i].new(token_string) if token.is_a?(Tokens::Unknown) raise Keisan::Exceptions::TokenizingError.new("Unexpected token: \"#{token.string}\"") end token end end