class Rbscmlex::Lexer

A lexical analyzer for Scheme program. It returns an Array instance of tokens. The representation of each token is depends on the specification of type, which is specified to the `new` method.

Currently, 3 types of the representation are available.

1. a struct Token,
2. a Hash object,
3. a JSON string.

Public Class Methods

new(obj, form: TOKEN_DEFAULT_FORM) click to toggle source
# File lib/rbscmlex/lexer.rb, line 30
def initialize(obj, form: TOKEN_DEFAULT_FORM)
  set_form(form)
  init_pos
  case obj
  when String
    # obj must be source program of Scheme.
    @tokens = tokenize(obj)
  when Array
    # obj might be an array of tokens.
    input_form = detect_form(obj[0])
    case input_form
    when :hash, :json, :token
      @tokens = read_tokens(obj, form: input_form)
    else
      raise InvalidConversionTypeError, "cannot convert #{obj[0]} as token"
    end
  else
    raise InvalidConversionTypeError, "cannot convert #{obj} as tokens"
  end
end

Public Instance Methods

[](index) click to toggle source
# File lib/rbscmlex/lexer.rb, line 51
def [](index)
  token = @tokens[index]
  token and convert(token)
end
current_token() click to toggle source
# File lib/rbscmlex/lexer.rb, line 75
def current_token
  self[@current_pos]
end
each() { |convert(tk)| ... } click to toggle source
# File lib/rbscmlex/lexer.rb, line 56
def each(&blk)
  if block_given?
    @tokens.each { |tk|
      yield convert(tk)
    }
    self
  else
    to_a.to_enum
  end
end
next_token(offset = 0) click to toggle source
# File lib/rbscmlex/lexer.rb, line 79
def next_token(offset = 0)
  check_pos(offset)
  skip_token(offset)
  self[@current_pos]
end
peek_token(offset = 0) click to toggle source
# File lib/rbscmlex/lexer.rb, line 85
def peek_token(offset = 0)
  # Since `peek_token` does not modify the position to read, raise
  # StopIteration only if the next position truly exceed the
  # bound.
  check_pos(0)
  self[@next_pos + offset]
end
rewind() click to toggle source
# File lib/rbscmlex/lexer.rb, line 100
def rewind
  init_pos
  self
end
size() click to toggle source
# File lib/rbscmlex/lexer.rb, line 71
def size
  @tokens.size
end
skip_token(offset = 0) click to toggle source
# File lib/rbscmlex/lexer.rb, line 93
def skip_token(offset = 0)
  check_pos(offset)
  @current_pos = @next_pos + offset
  @next_pos += (1 + offset)
  nil
end
to_a() click to toggle source
# File lib/rbscmlex/lexer.rb, line 67
def to_a
  convert_all(@tokens)
end