class GenericLexer
Public Class Methods
new()
click to toggle source
# File lib/crokus/generic_lexer.rb, line 7 def initialize @rules = [] @rules << [:newline,/[\n]/] end
Public Instance Methods
get_token()
click to toggle source
# File lib/crokus/generic_lexer.rb, line 36 def get_token linecol=position() @rules.each do |rule, regexp| val = @ssc.scan(regexp) return Token.new([rule, val, linecol]) if val end raise "lexing error line #{linecol.first} around '#{@ssc.peek(10)}' " end
ignore(pattern)
click to toggle source
# File lib/crokus/generic_lexer.rb, line 12 def ignore pattern @rules << [:skip,pattern] end
keyword(str)
click to toggle source
# File lib/crokus/generic_lexer.rb, line 16 def keyword str @rules.unshift [str.to_sym,/#{str}\b/] end
next_token()
click to toggle source
# File lib/crokus/generic_lexer.rb, line 30 def next_token return [nil,nil,nil] if @ssc.empty? tok = get_token return (tok.is? :skip) ? next_token : tok end
open(code)
click to toggle source
# File lib/crokus/generic_lexer.rb, line 25 def open code @ssc = StringScanner.new code @line=0 end
position()
click to toggle source
# File lib/crokus/generic_lexer.rb, line 45 def position if @ssc.bol? @line+=1 @old_pos=@ssc.pos end [@line,@ssc.pos-@old_pos+1] end
token(h_token_pattern)
click to toggle source
# File lib/crokus/generic_lexer.rb, line 20 def token h_token_pattern token,pattern=*h_token_pattern.to_a.first @rules << [token, pattern] end
tokenize(code)
click to toggle source
# File lib/crokus/generic_lexer.rb, line 53 def tokenize code open(code) tokens=[] tokens << next_token() while not @ssc.eos? tokens end