class GenericLexer

# File lib/generic_lexer.rb, line 7
def initialize
  @rules = []
  @rules << [:newline,/[\n]/]
end

# File lib/generic_lexer.rb, line 36
def get_token
  linecol=position()
  @rules.each do |rule, regexp|
    val = @ssc.scan(regexp)
    return Token.new([rule, val, linecol]) if val
  end
  raise  "lexing error line #{linecol.first} around : ...'#{@ssc.peek(5)}'... "
end

# File lib/generic_lexer.rb, line 12
def ignore pattern
  @rules << [:skip,pattern]
end

# File lib/generic_lexer.rb, line 16
def keyword str
  @rules.unshift [str.to_sym,/#{str}\b/i]
end

# File lib/generic_lexer.rb, line 30
def next_token
  return [nil,nil,nil] if @ssc.empty?
  tok = get_token
  return (tok.is_a? :skip) ? next_token : tok
end

# File lib/generic_lexer.rb, line 25
def open code
  @ssc = StringScanner.new code
  @line=0
end

# File lib/generic_lexer.rb, line 45
def position
  if @ssc.bol?
    @line+=1
    @old_pos=@ssc.pos
  end
  [@line,@ssc.pos-@old_pos+1]
end

# File lib/generic_lexer.rb, line 20
def token hash
  token,pattern=*hash.to_a.flatten
  @rules << [token, pattern]
end

# File lib/generic_lexer.rb, line 53
def tokenize code
  open(code)
  tokens=[]
  tokens << next_token() while not @ssc.eos?
  # while not @ssc.eos?
  #   tokens << (p next_token)
  # end #usefull for debug
  tokens
end