class Antlr4::Runtime::BufferedTokenStream
Attributes
token_source[R]
tokens[R]
Public Class Methods
new(token_source)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 7 def initialize(token_source) raise NilPointerException, 'token_source cannot be nil' if token_source.nil? @token_source = token_source @tokens = [] @ptr = -1 @fetched_eof = false end
Public Instance Methods
adjust_seek_index(i)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 145 def adjust_seek_index(i) i end
consume()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 43 def consume skip_eof_check = false if @ptr >= 0 if @fetched_eof # the last token in tokens is EOF. skip check if p indexes any # fetched token except the last. skip_eof_check = @ptr < @tokens.length - 1 else # no EOF token in tokens. skip check if p indexes a fetched token. skip_eof_check = @ptr < @tokens.length end else # not yet initialized skip_eof_check = false end if !skip_eof_check && la(1) == EOF raise IllegalStateException, 'cannot consume EOF' end @ptr = adjust_seek_index(@ptr + 1) if sync(@ptr + 1) end
fetch(n)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 75 def fetch(n) return 0 if @fetched_eof i = 0 while i < n t = @token_source.next_token t.token_index = @tokens.length if t.is_a? WritableToken @tokens << t if t.type == Token::EOF @fetched_eof = true return i + 1 end i += 1 end n end
fill()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 326 def fill lazy_init block_size = 1000 loop do fetched = fetch(block_size) return if fetched < block_size end end
filter_for_channel(from, to, channel)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 269 def filter_for_channel(from, to, channel) hidden = [] i = from while i <= to t = @tokens[i] if channel == -1 hidden << t if t.channel != Lexer::DEFAULT_TOKEN_CHANNEL else hidden << t if t.channel == channel end i += 1 end return nil if hidden.empty? hidden end
get(i)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 93 def get(i) if i < 0 || i >= @tokens.length raise IndexOutOfBoundsException, 'token index ' + i + ' out of range 0..' + (@tokens.length - 1) end @tokens[i] end
get_list(start, stop)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 101 def get_list(start, stop) return nil if start < 0 || stop < 0 lazy_init subset = [] stop = @tokens.length - 1 if stop >= @tokens.length i = start while i <= stop t = @tokens[i] break if t.type == Token::EOF subset << t i += 1 end subset end
get_tokens2(start, stop, ttype)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 187 def get_tokens2(start, stop, ttype) s = Set.new s << ttype tokens1(start, stop, s) end
index()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 18 def index @ptr end
la(i)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 118 def la(i) lt(i).type end
lazy_init()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 149 def lazy_init setup if @ptr == -1 end
lb(k)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 122 def lb(k) return nil if (@ptr - k) < 0 @tokens[@ptr - k] end
lt(k)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 128 def lt(k) lazy_init return nil if k == 0 return lb(-k) if k < 0 i = @ptr + k - 1 sync(i) if i >= @tokens.length # return EOF token # EOF must be last token return @tokens.get(@tokens.length - 1) end # if ( i>range ) range = i @tokens[i] end
mark()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 22 def mark 0 end
next_token_on_channel(i, channel)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 193 def next_token_on_channel(i, channel) sync(i) return size - 1 if i >= size token = @tokens[i] while token.channel != channel return i if token.type == Token::EOF i += 1 sync(i) token = @tokens[i] end i end
previous_token_on_channel(i, channel)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 209 def previous_token_on_channel(i, channel) sync(i) if i >= size # the EOF token is on every channel return size - 1 end while i >= 0 token = @tokens[i] return i if token.type == Token::EOF || token.channel == channel i -= 1 end i end
release(marker)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 26 def release(marker) ; end
reset()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 30 def reset seek(0) end
seek(index)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 34 def seek(index) lazy_init @ptr = adjust_seek_index(index) end
set_token_source(tokenSource)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 158 def set_token_source(tokenSource) @token_source = tokenSource @tokens.clear @ptr = -1 @fetched_eof = false end
setup()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 153 def setup sync(0) @ptr = adjust_seek_index(0) end
size()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 39 def size @tokens.length end
source_name()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 286 def source_name @token_source.get_source_name end
sync(i)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 64 def sync(i) n = i - @tokens.length + 1 # how many more elements we need? if n > 0 fetched = fetch(n) return fetched >= n end true end
text()
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 290 def text text2(Interval.of(0, size - 1)) end
text2(interval)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 294 def text2(interval) start = interval.a stop = interval.b return '' if start < 0 || stop < 0 fill stop = @tokens.length - 1 if stop >= @tokens.length buf = '' i = start while i <= stop t = @tokens[i] break if t.type == Token::EOF buf << t.text i += 1 end buf end
text3(ctx)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 314 def text3(ctx) text2(ctx.source_interval) end
text4(start, stop)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 318 def text4(start, stop) if !start.nil? && !stop.nil? return text2(Interval.of(start.token_index, stop.token_index)) end '' end
tokens1(start, stop, types = nil)
click to toggle source
# File lib/antlr4/runtime/buffered_token_stream.rb, line 167 def tokens1(start, stop, types = nil) lazy_init if start < 0 || stop >= @tokens.length || stop < 0 || start >= @tokens.length raise IndexOutOfBoundsException, 'start ' + start + ' or stop ' + stop + ' not in 0..' + (@tokens.length - 1) end return nil if start > stop # list = tokens[start:stop]:T t, t.getType() in typesend filtered_tokens = [] i = start while i <= stop t = @tokens[i] filtered_tokens << t if types.nil? || types.include?(t.type) i += 1 end filtered_tokens = nil if filtered_tokens.empty? filtered_tokens end