class Omnium::Lexer::Core

The lexer returns tokens for a given text.

Constants

ALPHA
COMMENT
DECIMAL_POINT
IDENTIFIER
INTEGER
NEWLINE
WHITESPACE

Public Class Methods

new(text) click to toggle source
# File lib/omnium/lexer/core.rb, line 23
def initialize(text)
  @text = text
  @pointer = 0
end

Public Instance Methods

next_token() click to toggle source
# File lib/omnium/lexer/core.rb, line 28
def next_token
  ignore
  return new_eof_token if eos?

  (@pointer...@text.length).each do
    return tokenise
  end
end

Private Instance Methods

advance(n = 1) click to toggle source
# File lib/omnium/lexer/core.rb, line 52
def advance(n = 1)
  @pointer += n
end
assignment?() click to toggle source
# File lib/omnium/lexer/core.rb, line 116
def assignment?
  colon = assignment_token(:value)[0]
  equals = assignment_token(:value)[1]

  character == colon && peek == equals
end
character() click to toggle source
# File lib/omnium/lexer/core.rb, line 56
def character
  @character = @text[@pointer]
end
comment?() click to toggle source
# File lib/omnium/lexer/core.rb, line 70
def comment?
  character == COMMENT
end
decimal?() click to toggle source
# File lib/omnium/lexer/core.rb, line 78
def decimal?
  character == DECIMAL_POINT
end
eos?() click to toggle source
# File lib/omnium/lexer/core.rb, line 48
def eos?
  @pointer > @text.length - 1
end
ignore() click to toggle source
# File lib/omnium/lexer/core.rb, line 39
def ignore
  advance while whitespace? || newline?

  if comment?
    advance until newline? # comment text
    ignore if whitespace? || newline? # skip any other junk
  end
end
newline?() click to toggle source
# File lib/omnium/lexer/core.rb, line 74
def newline?
  character == NEWLINE
end
number() click to toggle source
# File lib/omnium/lexer/core.rb, line 82
def number
  result = ''

  while character =~ INTEGER
    result += character
    advance
  end

  if decimal?
    result += character
    advance

    while character =~ INTEGER
      result += character
      advance
    end

    return result.to_f
  end

  result.to_i
end
number_token(num) click to toggle source
# File lib/omnium/lexer/core.rb, line 131
def number_token(num)
  return new_integer_token(num) if num.is_a? Integer

  new_real_token(num)
end
peek() click to toggle source
# File lib/omnium/lexer/core.rb, line 60
def peek
  return nil if eos?

  @text[@pointer + 1]
end
reserved_keyword() click to toggle source
# File lib/omnium/lexer/core.rb, line 105
def reserved_keyword
  result = ''

  while character =~ IDENTIFIER
    result += character
    advance
  end

  result
end
tokenise() click to toggle source
# File lib/omnium/lexer/core.rb, line 137
def tokenise
  if character =~ ALPHA
    word_token(reserved_keyword)
  elsif character =~ INTEGER
    number_token(number)
  elsif plus?
    advance
    new_plus_token
  elsif minus?
    advance
    new_minus_token
  elsif multiply?
    advance
    new_multiply_token
  elsif divide?
    advance
    new_divide_token
  elsif left_parenthesis?
    advance
    new_left_parenthesis_token
  elsif right_parenthesis?
    advance
    new_right_parenthesis_token
  elsif assignment?
    advance(2)
    new_assignment_token
  elsif semicolon?
    advance
    new_semicolon_token
  elsif colon?
    advance
    new_colon_token
  elsif comma?
    advance
    new_comma_token
  elsif dot?
    advance
    new_dot_token
  else
    raise(LexerError, "Error tokenising '#{character}' at position #{@pointer}")
  end
end
whitespace?() click to toggle source
# File lib/omnium/lexer/core.rb, line 66
def whitespace?
  character == WHITESPACE
end
word_token(word) click to toggle source
# File lib/omnium/lexer/core.rb, line 123
def word_token(word)
  if RESERVED_KEYWORDS.include?(word.intern)
    send("new_#{word}_token")
  else
    new_identifier_token(word)
  end
end