class Liquid::Lexer
Constants
- COMPARISON_OPERATOR
- DOTDOT
- DOUBLE_STRING_LITERAL
- IDENTIFIER
- NUMBER_LITERAL
- SINGLE_STRING_LITERAL
- SPECIALS
- WHITESPACE_OR_NOTHING
Public Class Methods
new(input)
click to toggle source
# File lib/liquid/lexer.rb, line 24 def initialize(input) @ss = StringScanner.new(input) end
Public Instance Methods
tokenize()
click to toggle source
# File lib/liquid/lexer.rb, line 28 def tokenize @output = [] until @ss.eos? @ss.skip(WHITESPACE_OR_NOTHING) break if @ss.eos? tok = case when t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t] when t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t] when t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t] when t = @ss.scan(NUMBER_LITERAL) then [:number, t] when t = @ss.scan(IDENTIFIER) then [:id, t] when t = @ss.scan(DOTDOT) then [:dotdot, t] else c = @ss.getch if s = SPECIALS[c] [s, c] else raise SyntaxError, "Unexpected character #{c}" end end @output << tok end @output << [:end_of_string] end