class Sparkql::Lexer

Attributes

block_group_identifier[RW]
current_token_value[R]
last_field[R]
level[RW]
token_index[R]

Public Class Methods

new(str) click to toggle source
Calls superclass method
# File lib/sparkql/lexer.rb, line 10
def initialize(str)
  str.freeze
  super(str, false) # DO NOT dup str
  @level = 0
  @block_group_identifier = 0
  @expression_count = 0
end

Public Instance Methods

check_keywords(value) click to toggle source
# File lib/sparkql/lexer.rb, line 100
def check_keywords(value)
  result = check_reserved_words(value)
  if result.first == :UNKNOWN
    result = [:KEYWORD,value]
  end
  result
end
check_reserved_words(value) click to toggle source
# File lib/sparkql/lexer.rb, line 66
def check_reserved_words(value)
  u_value = value.capitalize
  if OPERATORS.include?(u_value)
    [:OPERATOR,u_value]
  elsif RANGE_OPERATOR == u_value
    [:RANGE_OPERATOR,u_value]
  elsif CONJUNCTIONS.include?(u_value)
    [:CONJUNCTION,u_value]
  elsif UNARY_CONJUNCTIONS.include?(u_value)
    [:UNARY_CONJUNCTION,u_value]
  elsif ADD == u_value
    [:ADD, u_value]
  elsif SUB == u_value
    [:SUB, u_value]
  elsif MUL == u_value
    [:MUL, u_value]
  elsif DIV == u_value
    [:DIV, u_value]
  elsif MOD == u_value
    [:MOD, u_value]
  else
    [:UNKNOWN, "ERROR: '#{self.string}'"]
  end
end
check_standard_fields(value) click to toggle source
# File lib/sparkql/lexer.rb, line 91
def check_standard_fields(value)
  result = check_reserved_words(value)
  if result.first == :UNKNOWN
    @last_field = value
    result = [:STANDARD_FIELD,value]
  end
  result
end
leveldown() click to toggle source
# File lib/sparkql/lexer.rb, line 113
def leveldown
  @level -= 1
end
levelup() click to toggle source
# File lib/sparkql/lexer.rb, line 108
def levelup
  @level += 1
  @block_group_identifier += 1
end
literal(symbol, value) click to toggle source
# File lib/sparkql/lexer.rb, line 117
def literal(symbol, value)
  node = {
    :type => symbol.to_s.downcase.to_sym,
    :value => value
  }
  [symbol, node]
end
shift() click to toggle source

Lookup the next matching token

# File lib/sparkql/lexer.rb, line 19
def shift
  @token_index = self.pos

  token = case
    when @current_token_value = scan(SPACE)
      [:SPACE, @current_token_value]
    when @current_token_value = scan(LPAREN)
      levelup
      [:LPAREN, @current_token_value]
    when @current_token_value = scan(RPAREN)
      # leveldown: do this after parsing group
      [:RPAREN, @current_token_value]
    when @current_token_value = scan(/\,/)
      [:COMMA,@current_token_value]
    when @current_token_value = scan(NULL)
      literal :NULL, "NULL"
    when @current_token_value = scan(STANDARD_FIELD)
      check_standard_fields(@current_token_value)
    when @current_token_value = scan(DATETIME)
      literal :DATETIME, @current_token_value
    when @current_token_value = scan(DATE)
      literal :DATE, @current_token_value
    when @current_token_value = scan(TIME)
      literal :TIME, @current_token_value
    when @current_token_value = scan(DECIMAL)
      literal :DECIMAL, @current_token_value
    when @current_token_value = scan(INTEGER)
      literal :INTEGER, @current_token_value
    when @current_token_value = scan(/\-/)
      [:UMINUS, @current_token_value]
    when @current_token_value = scan(CHARACTER)
      literal :CHARACTER, @current_token_value
    when @current_token_value = scan(BOOLEAN)
      literal :BOOLEAN, @current_token_value
    when @current_token_value = scan(KEYWORD)
      check_keywords(@current_token_value)
    when @current_token_value = scan(CUSTOM_FIELD)
      [:CUSTOM_FIELD,@current_token_value]
    when eos?
      [false, false] # end of file, \Z don't work with StringScanner
    else
      [:UNKNOWN, "ERROR: '#{self.string}'"]
  end

  token.freeze
end