class Rbscmlex::Lexer
A lexical analyzer for Scheme program. It returns an Array instance of tokens. The representation of each token is depends on the specification of type, which is specified to the `new` method.
Currently, 3 types of the representation are available.
1. a struct Token, 2. a Hash object, 3. a JSON string.
Public Class Methods
new(obj, form: TOKEN_DEFAULT_FORM)
click to toggle source
# File lib/rbscmlex/lexer.rb, line 30 def initialize(obj, form: TOKEN_DEFAULT_FORM) set_form(form) init_pos case obj when String # obj must be source program of Scheme. @tokens = tokenize(obj) when Array # obj might be an array of tokens. input_form = detect_form(obj[0]) case input_form when :hash, :json, :token @tokens = read_tokens(obj, form: input_form) else raise InvalidConversionTypeError, "cannot convert #{obj[0]} as token" end else raise InvalidConversionTypeError, "cannot convert #{obj} as tokens" end end
Public Instance Methods
[](index)
click to toggle source
# File lib/rbscmlex/lexer.rb, line 51 def [](index) token = @tokens[index] token and convert(token) end
current_token()
click to toggle source
# File lib/rbscmlex/lexer.rb, line 75 def current_token self[@current_pos] end
each() { |convert(tk)| ... }
click to toggle source
# File lib/rbscmlex/lexer.rb, line 56 def each(&blk) if block_given? @tokens.each { |tk| yield convert(tk) } self else to_a.to_enum end end
next_token(offset = 0)
click to toggle source
# File lib/rbscmlex/lexer.rb, line 79 def next_token(offset = 0) check_pos(offset) skip_token(offset) self[@current_pos] end
peek_token(offset = 0)
click to toggle source
# File lib/rbscmlex/lexer.rb, line 85 def peek_token(offset = 0) # Since `peek_token` does not modify the position to read, raise # StopIteration only if the next position truly exceed the # bound. check_pos(0) self[@next_pos + offset] end
rewind()
click to toggle source
# File lib/rbscmlex/lexer.rb, line 100 def rewind init_pos self end
size()
click to toggle source
# File lib/rbscmlex/lexer.rb, line 71 def size @tokens.size end
skip_token(offset = 0)
click to toggle source
# File lib/rbscmlex/lexer.rb, line 93 def skip_token(offset = 0) check_pos(offset) @current_pos = @next_pos + offset @next_pos += (1 + offset) nil end
to_a()
click to toggle source
# File lib/rbscmlex/lexer.rb, line 67 def to_a convert_all(@tokens) end