class GenericLexer
Public Class Methods
new()
click to toggle source
# File lib/generic_lexer.rb, line 7 def initialize @rules = [] @rules << [:newline,/[\n]/] end
Public Instance Methods
get_token()
click to toggle source
# File lib/generic_lexer.rb, line 36 def get_token linecol=position() @rules.each do |rule, regexp| val = @ssc.scan(regexp) return Token.new([rule, val, linecol]) if val end raise "lexing error line #{linecol.first} around : ...'#{@ssc.peek(5)}'... " end
ignore(pattern)
click to toggle source
# File lib/generic_lexer.rb, line 12 def ignore pattern @rules << [:skip,pattern] end
keyword(str)
click to toggle source
# File lib/generic_lexer.rb, line 16 def keyword str @rules.unshift [str.to_sym,/#{str}\b/i] end
next_token()
click to toggle source
# File lib/generic_lexer.rb, line 30 def next_token return [nil,nil,nil] if @ssc.empty? tok = get_token return (tok.is_a? :skip) ? next_token : tok end
open(code)
click to toggle source
# File lib/generic_lexer.rb, line 25 def open code @ssc = StringScanner.new code @line=0 end
position()
click to toggle source
# File lib/generic_lexer.rb, line 45 def position if @ssc.bol? @line+=1 @old_pos=@ssc.pos end [@line,@ssc.pos-@old_pos+1] end
token(hash)
click to toggle source
# File lib/generic_lexer.rb, line 20 def token hash token,pattern=*hash.to_a.flatten @rules << [token, pattern] end
tokenize(code)
click to toggle source
# File lib/generic_lexer.rb, line 53 def tokenize code open(code) tokens=[] tokens << next_token() while not @ssc.eos? # while not @ssc.eos? # tokens << (p next_token) # end #usefull for debug tokens end