class Halunke::Tokenizer
Attributes
_tokenizer_actions[RW]
_tokenizer_eof_trans[RW]
_tokenizer_from_state_actions[RW]
_tokenizer_index_offsets[RW]
_tokenizer_key_offsets[RW]
_tokenizer_range_lengths[RW]
_tokenizer_single_lengths[RW]
_tokenizer_to_state_actions[RW]
_tokenizer_trans_actions[RW]
_tokenizer_trans_keys[RW]
_tokenizer_trans_targs[RW]
tokenizer_en_main[RW]
tokenizer_error[RW]
tokenizer_first_final[RW]
tokenizer_start[RW]
Public Class Methods
new()
click to toggle source
# File lib/halunke/tokenizer.rb, line 11 def initialize # line 14 "lib/halunke/tokenizer.rb" class << self attr_accessor :_tokenizer_actions private :_tokenizer_actions, :_tokenizer_actions= end self._tokenizer_actions = [ 0, 1, 0, 1, 1, 1, 2, 1, 5, 1, 6, 1, 7, 1, 8, 1, 9, 1, 10, 1, 11, 1, 12, 1, 13, 1, 14, 1, 15, 1, 16, 1, 17, 1, 18, 1, 19, 1, 20, 1, 21, 1, 22, 1, 23, 1, 24, 2, 2, 3, 2, 2, 4 ] class << self attr_accessor :_tokenizer_key_offsets private :_tokenizer_key_offsets, :_tokenizer_key_offsets= end self._tokenizer_key_offsets = [ 0, 1, 3, 29, 30, 35, 37, 39, 42, 44, 45, 46 ] class << self attr_accessor :_tokenizer_trans_keys private :_tokenizer_trans_keys, :_tokenizer_trans_keys= end self._tokenizer_trans_keys = [ 34, 48, 57, 32, 34, 39, 40, 41, 42, 43, 45, 47, 64, 91, 93, 95, 123, 124, 125, 9, 13, 48, 57, 60, 62, 65, 90, 97, 122, 34, 95, 65, 90, 97, 122, 42, 47, 48, 57, 46, 48, 57, 48, 57, 42, 91, 95, 65, 90, 97, 122, 0 ] class << self attr_accessor :_tokenizer_single_lengths private :_tokenizer_single_lengths, :_tokenizer_single_lengths= end self._tokenizer_single_lengths = [ 1, 0, 16, 1, 1, 2, 0, 1, 0, 1, 1, 1 ] class << self attr_accessor :_tokenizer_range_lengths private :_tokenizer_range_lengths, :_tokenizer_range_lengths= end self._tokenizer_range_lengths = [ 0, 1, 5, 0, 2, 0, 1, 1, 1, 0, 0, 2 ] class << self attr_accessor :_tokenizer_index_offsets private :_tokenizer_index_offsets, :_tokenizer_index_offsets= end self._tokenizer_index_offsets = [ 0, 2, 4, 26, 28, 32, 35, 37, 40, 42, 44, 46 ] class << self attr_accessor :_tokenizer_trans_targs private :_tokenizer_trans_targs, :_tokenizer_trans_targs= end self._tokenizer_trans_targs = [ 2, 0, 8, 2, 2, 3, 4, 2, 2, 5, 6, 6, 9, 10, 2, 2, 11, 2, 2, 2, 2, 7, 2, 11, 11, 2, 2, 0, 4, 4, 4, 2, 2, 2, 2, 7, 2, 1, 7, 2, 8, 2, 2, 2, 2, 2, 11, 11, 11, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0 ] class << self attr_accessor :_tokenizer_trans_actions private :_tokenizer_trans_actions, :_tokenizer_trans_actions= end self._tokenizer_trans_actions = [ 7, 0, 0, 41, 31, 5, 50, 11, 13, 0, 0, 0, 0, 0, 19, 21, 0, 15, 29, 17, 31, 5, 9, 0, 0, 33, 7, 0, 47, 47, 47, 45, 9, 27, 37, 5, 37, 0, 5, 35, 0, 35, 25, 37, 23, 37, 0, 0, 0, 37, 43, 41, 39, 45, 37, 37, 35, 35, 37, 37, 37, 0 ] class << self attr_accessor :_tokenizer_to_state_actions private :_tokenizer_to_state_actions, :_tokenizer_to_state_actions= end self._tokenizer_to_state_actions = [ 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] class << self attr_accessor :_tokenizer_from_state_actions private :_tokenizer_from_state_actions, :_tokenizer_from_state_actions= end self._tokenizer_from_state_actions = [ 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] class << self attr_accessor :_tokenizer_eof_trans private :_tokenizer_eof_trans, :_tokenizer_eof_trans= end self._tokenizer_eof_trans = [ 51, 52, 0, 53, 54, 61, 61, 58, 58, 61, 61, 61 ] class << self attr_accessor :tokenizer_start end self.tokenizer_start = 2; class << self attr_accessor :tokenizer_first_final end self.tokenizer_first_final = 2; class << self attr_accessor :tokenizer_error end self.tokenizer_error = -1; class << self attr_accessor :tokenizer_en_main end self.tokenizer_en_main = 2; # line 47 "lib/halunke/tokenizer.rl" @tokens = [] end
Public Instance Methods
tokenize(data)
click to toggle source
# File lib/halunke/tokenizer.rb, line 158 def tokenize(data) data.chomp! eof = data.length # line 164 "lib/halunke/tokenizer.rb" begin p ||= 0 pe ||= data.length cs = tokenizer_start ts = nil te = nil act = 0 end # line 55 "lib/halunke/tokenizer.rl" # line 176 "lib/halunke/tokenizer.rb" begin _klen, _trans, _keys, _acts, _nacts = nil _goto_level = 0 _resume = 10 _eof_trans = 15 _again = 20 _test_eof = 30 _out = 40 while true _trigger_goto = false if _goto_level <= 0 if p == pe _goto_level = _test_eof next end end if _goto_level <= _resume _acts = _tokenizer_from_state_actions[cs] _nacts = _tokenizer_actions[_acts] _acts += 1 while _nacts > 0 _nacts -= 1 _acts += 1 case _tokenizer_actions[_acts - 1] when 1 then # line 1 "NONE" begin ts = p end # line 206 "lib/halunke/tokenizer.rb" end # from state action switch end if _trigger_goto next end _keys = _tokenizer_key_offsets[cs] _trans = _tokenizer_index_offsets[cs] _klen = _tokenizer_single_lengths[cs] _break_match = false begin if _klen > 0 _lower = _keys _upper = _keys + _klen - 1 loop do break if _upper < _lower _mid = _lower + ( (_upper - _lower) >> 1 ) if data[p].ord < _tokenizer_trans_keys[_mid] _upper = _mid - 1 elsif data[p].ord > _tokenizer_trans_keys[_mid] _lower = _mid + 1 else _trans += (_mid - _keys) _break_match = true break end end # loop break if _break_match _keys += _klen _trans += _klen end _klen = _tokenizer_range_lengths[cs] if _klen > 0 _lower = _keys _upper = _keys + (_klen << 1) - 2 loop do break if _upper < _lower _mid = _lower + (((_upper-_lower) >> 1) & ~1) if data[p].ord < _tokenizer_trans_keys[_mid] _upper = _mid - 2 elsif data[p].ord > _tokenizer_trans_keys[_mid+1] _lower = _mid + 2 else _trans += ((_mid - _keys) >> 1) _break_match = true break end end # loop break if _break_match _trans += _klen end end while false end if _goto_level <= _eof_trans cs = _tokenizer_trans_targs[_trans] if _tokenizer_trans_actions[_trans] != 0 _acts = _tokenizer_trans_actions[_trans] _nacts = _tokenizer_actions[_acts] _acts += 1 while _nacts > 0 _nacts -= 1 _acts += 1 case _tokenizer_actions[_acts - 1] when 2 then # line 1 "NONE" begin te = p+1 end when 3 then # line 24 "lib/halunke/tokenizer.rl" begin act = 3; end when 4 then # line 37 "lib/halunke/tokenizer.rl" begin act = 16; end when 5 then # line 23 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:STRING, data[ts+1...te-1], ts, te - 1) end end when 6 then # line 25 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:BAREWORD, data[ts...te], ts, te - 1) end end when 7 then # line 26 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:OPEN_PAREN, data[ts...te], ts, te - 1) end end when 8 then # line 27 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:CLOSE_PAREN, data[ts...te], ts, te - 1) end end when 9 then # line 28 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:OPEN_CURLY, data[ts...te], ts, te - 1) end end when 10 then # line 29 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:CLOSE_CURLY, data[ts...te], ts, te - 1) end end when 11 then # line 30 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:OPEN_BRACKET, data[ts...te], ts, te - 1) end end when 12 then # line 31 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:CLOSE_BRACKET, data[ts...te], ts, te - 1) end end when 13 then # line 32 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:OPEN_DICT_BRACKET, data[ts...te], ts, te - 1) end end when 14 then # line 33 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:START_COMMENT, data[ts...te], ts, te - 1) end end when 15 then # line 34 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:END_COMMENT, data[ts...te], ts, te - 1) end end when 16 then # line 35 "lib/halunke/tokenizer.rl" begin te = p+1 begin emit(:BAR, data[ts...te], ts, te - 1) end end when 17 then # line 36 "lib/halunke/tokenizer.rl" begin te = p+1 end when 18 then # line 37 "lib/halunke/tokenizer.rl" begin te = p+1 begin raise "Could not lex '#{ data[ts...te] }'" end end when 19 then # line 22 "lib/halunke/tokenizer.rl" begin te = p p = p - 1; begin emit(:NUMBER, data[ts...te].to_r, ts, te - 1) end end when 20 then # line 25 "lib/halunke/tokenizer.rl" begin te = p p = p - 1; begin emit(:BAREWORD, data[ts...te], ts, te - 1) end end when 21 then # line 37 "lib/halunke/tokenizer.rl" begin te = p p = p - 1; begin raise "Could not lex '#{ data[ts...te] }'" end end when 22 then # line 22 "lib/halunke/tokenizer.rl" begin begin p = ((te))-1; end begin emit(:NUMBER, data[ts...te].to_r, ts, te - 1) end end when 23 then # line 37 "lib/halunke/tokenizer.rl" begin begin p = ((te))-1; end begin raise "Could not lex '#{ data[ts...te] }'" end end when 24 then # line 1 "NONE" begin case act when 3 then begin begin p = ((te))-1; end emit(:UNASSIGNED_BAREWORD, data[ts+1 ...te], ts, te - 1) end when 16 then begin begin p = ((te))-1; end raise "Could not lex '#{ data[ts...te] }'" end end end # line 410 "lib/halunke/tokenizer.rb" end # action switch end end if _trigger_goto next end end if _goto_level <= _again _acts = _tokenizer_to_state_actions[cs] _nacts = _tokenizer_actions[_acts] _acts += 1 while _nacts > 0 _nacts -= 1 _acts += 1 case _tokenizer_actions[_acts - 1] when 0 then # line 1 "NONE" begin ts = nil; end # line 430 "lib/halunke/tokenizer.rb" end # to state action switch end if _trigger_goto next end p += 1 if p != pe _goto_level = _resume next end end if _goto_level <= _test_eof if p == eof if _tokenizer_eof_trans[cs] > 0 _trans = _tokenizer_eof_trans[cs] - 1; _goto_level = _eof_trans next; end end end if _goto_level <= _out break end end end # line 56 "lib/halunke/tokenizer.rl" @tokens end
Private Instance Methods
emit(type, value, ts, te)
click to toggle source
# File lib/halunke/tokenizer.rb, line 463 def emit(type, value, ts, te) @tokens << [ type, [ value, ts, te ] ] end