class Keisan::Tokenizer

Constants

TOKEN_CLASSES
TOKEN_REGEX

Attributes

expression[R]
tokens[R]

Public Class Methods

new(expression) click to toggle source
# File lib/keisan/tokenizer.rb, line 26
def initialize(expression)
  @expression = expression

  portions = StringAndGroupParser.new(expression).portions.reject do |portion|
    portion.is_a? StringAndGroupParser::CommentPortion
  end

  @tokens = portions.inject([]) do |tokens, portion|
    case portion
    when StringAndGroupParser::StringPortion
      tokens << Tokens::String.new(portion.escaped_string)
    when StringAndGroupParser::GroupPortion
      tokens << Tokens::Group.new(portion.to_s)
    when StringAndGroupParser::OtherPortion
      scan = portion.to_s.scan(TOKEN_REGEX)
      tokens += tokenize!(scan)
    end

    tokens
  end
end

Private Instance Methods

tokenize!(scan) click to toggle source
# File lib/keisan/tokenizer.rb, line 50
def tokenize!(scan)
  scan.map do |scan_result|
    i = scan_result.find_index {|token| !token.nil?}
    token_string = scan_result[i]
    token = TOKEN_CLASSES[i].new(token_string)
    if token.is_a?(Tokens::Unknown)
      raise Keisan::Exceptions::TokenizingError.new("Unexpected token: \"#{token.string}\"")
    end
    token
  end
end