class Antlr4::Runtime::Lexer

Constants

DEFAULT_MODE
DEFAULT_TOKEN_CHANNEL
HIDDEN
MAX_CHAR_VALUE
MIN_CHAR_VALUE
MORE
SKIP

Attributes

_channel[RW]
_hit_eof[RW]
_input[RW]
_mode[RW]
_mode_stack[RW]
_text[RW]
_token_start_char_index[RW]
_token_start_char_position_in_line[RW]
_token_start_line[RW]
_type[RW]
token[RW]

Public Class Methods

new(input = nil) click to toggle source
Calls superclass method
# File lib/antlr4/runtime/lexer.rb, line 45
def initialize(input = nil)
  super()
  unless input.nil?
    @_input = input
    @_token_factory_source_pair = OpenStruct.new
    @_token_factory_source_pair.a = self
    @_token_factory_source_pair.b = input
  end
  @_mode_stack = []
  reset
  @_factory = CommonTokenFactory.instance
end

Public Instance Methods

all_tokens() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 200
def all_tokens
  tokens = []
  t = next_token
  while t.type != Token::EOF
    tokens << t
    t = next_token
  end
  tokens
end
char_error_display(c) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 251
def char_error_display(c)
  s = error_display_char(c)
  "'" + s + "'"
end
char_index() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 190
def char_index
  @_input.index
end
char_position_in_line() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 178
def char_position_in_line
  @_interp.char_position_in_line
end
emit(token = nil) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 159
def emit(token = nil)
  if !token.nil?
    @_token = token
  else
    @_token = @_factory.create(@_token_factory_source_pair, @_type, @_text, @_channel, @_token_start_char_index, char_index - 1, @_token_start_line, @_token_start_char_position_in_line)
  end
end
emit_eof() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 167
def emit_eof
  cpos = char_position_in_line
  eof = @_factory.create(@_token_factory_source_pair, Token::EOF, nil, Token::DEFAULT_CHANNEL, @_input.index, @_input.index - 1, line, cpos)
  emit(eof)
  eof
end
error_display(s) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 225
def error_display(s)
  buf = ''
  s.chars.each do |c|
    buf << error_display_char(c)
  end
  buf
end
error_display_char(c) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 233
def error_display_char(c)
  s = ''
  s << c
  case c
  when Token::EOF
    s = '<EOF>'
  when '\n'
    s = '\\n'
  when '\t'
    s = '\\t'
  when '\r'
    s = '\\r'
  else
    # type code here
  end
  s
end
input_stream(input) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 144
def input_stream(input)
  @_input = nil
  @_token_factory_source_pair = OpenStruct.new
  @_token_factory_source_pair.a = self
  @_token_factory_source_pair.b = @_input
  reset
  @_input = input
  @_token_factory_source_pair.a = self
  @_token_factory_source_pair.b = @_input
end
line() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 174
def line
  @_interp.line
end
mode(m) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 126
def mode(m)
  @_mode = m
end
more() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 122
def more
  @_type = MORE
end
next_token() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 66
def next_token
  if @_input.nil?
    raise IllegalStateException, 'next_token requires a non-nil input stream.'
  end

  # Mark start location in char stream so unbuffered streams are
  # guaranteed at least have text of current token
  token_start_marker = @_input.mark
  begin
    repeat_outer = true
    repeat_outer = next_token_inner while repeat_outer
    return @_token
  ensure # make sure we release marker after match or
    # unbuffered char stream will keep buffering
    @_input.release(token_start_marker)
  end
end
next_token_inner() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 84
def next_token_inner
  loop do
    if @_hit_eof
      emit_eof
      return false
    end

    @_token = nil
    @_channel = Token::DEFAULT_CHANNEL
    @_token_start_char_index = @_input.index
    @_token_start_char_position_in_line = @_interp.char_position_in_line
    @_token_start_line = @_interp.line
    @_text = nil
    loop do
      @_type = Token::INVALID_TYPE

      begin
        ttype = @_interp.match(@_input, @_mode)
      rescue LexerNoViableAltException => e
        notify_listeners(e) # report error
        recover1(e)
        ttype = SKIP
      end
      @_hit_eof = true if @_input.la(1) == IntStream::EOF
      @_type = ttype if @_type == Token::INVALID_TYPE
      return true if @_type == SKIP
      break if @_type != MORE
    end

    emit if @_token.nil?
    return false
  end
end
notify_listeners(e) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 217
def notify_listeners(e)
  text = @_input.text(Interval.of(@_token_start_char_index, @_input.index))
  msg = "token recognition error at: '" + error_display(text) + "'"

  listener = error_listener_dispatch
  listener.syntax_error(self, nil, @_token_start_line, @_token_start_char_position_in_line, msg, e)
end
pop_mode() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 136
def pop_mode
  raise EmptyStackException if @_mode_stack.empty?

  puts('popMode back to ' + @_mode_stack[-1]) if LexerATNSimulator.debug
  mode(@_mode_stack.pop)
  @_mode
end
push_mode(m) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 130
def push_mode(m)
  puts('pushMode ' + m) if LexerATNSimulator.debug
  @_mode_stack.push(@_mode)
  mode(m)
end
recover1(_e) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 210
def recover1(_e)
  if @_input.la(1) != IntStream::EOF
    # skip a char and begin again
    @_interp.consume(@_input)
  end
end
recover2(_re) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 256
def recover2(_re)
  @_input.consume
end
reset() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 25
def reset
  # wack Lexer state variables
  unless @_input.nil?
    @_input.seek(0) # rewind the input
  end
  @_token = nil
  @_type = Token::INVALID_TYPE
  @_channel = Token::DEFAULT_CHANNEL
  @_token_start_char_index = -1
  @_token_start_char_position_in_line = -1
  @_token_start_line = -1
  @_text = nil

  @_hit_eof = false
  @_mode = DEFAULT_MODE
  @_mode_stack.clear

  @_interp.reset unless @_interp.nil?
end
set_char_position_in_line(char_position_in_line) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 186
def set_char_position_in_line(char_position_in_line)
  @_interp.set_char_position_in_line(char_position_in_line)
end
set_line(line) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 182
def set_line(line)
  @_interp.set_line(line)
end
skip() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 118
def skip
  @_type = SKIP
end
source_name() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 155
def source_name
  @_input.get_source_name
end
text() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 194
def text
  return @_text unless @_text.nil?

  @_interp.text(@_input)
end
token_factory() click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 58
def token_factory
  @_factory
end
token_factory=(factory) click to toggle source
# File lib/antlr4/runtime/lexer.rb, line 62
def token_factory=(factory)
  @_factory = factory
end