class GenericLexer

Public Class Methods

new() click to toggle source
# File lib/crokus/generic_lexer.rb, line 7
def initialize
  @rules = []
  @rules << [:newline,/[\n]/]
end

Public Instance Methods

get_token() click to toggle source
# File lib/crokus/generic_lexer.rb, line 36
def get_token
  linecol=position()
  @rules.each do |rule, regexp|
    val = @ssc.scan(regexp)
    return Token.new([rule, val, linecol]) if val
  end
  raise  "lexing error line #{linecol.first} around '#{@ssc.peek(10)}' "
end
ignore(pattern) click to toggle source
# File lib/crokus/generic_lexer.rb, line 12
def ignore pattern
  @rules << [:skip,pattern]
end
keyword(str) click to toggle source
# File lib/crokus/generic_lexer.rb, line 16
def keyword str
  @rules.unshift [str.to_sym,/#{str}\b/]
end
next_token() click to toggle source
# File lib/crokus/generic_lexer.rb, line 30
def next_token
  return [nil,nil,nil] if @ssc.empty?
  tok = get_token
  return (tok.is? :skip) ? next_token : tok
end
open(code) click to toggle source
# File lib/crokus/generic_lexer.rb, line 25
def open code
  @ssc = StringScanner.new code
  @line=0
end
position() click to toggle source
# File lib/crokus/generic_lexer.rb, line 45
def position
  if @ssc.bol?
    @line+=1
    @old_pos=@ssc.pos
  end
  [@line,@ssc.pos-@old_pos+1]
end
token(h_token_pattern) click to toggle source
# File lib/crokus/generic_lexer.rb, line 20
def token h_token_pattern
  token,pattern=*h_token_pattern.to_a.first
  @rules << [token, pattern]
end
tokenize(code) click to toggle source
# File lib/crokus/generic_lexer.rb, line 53
def tokenize code
  open(code)
  tokens=[]
  tokens << next_token() while not @ssc.eos?
  tokens
end