class Parser::LexerStrings
line 3 “lib/parser/lexer-strings.rl”
Constants
- ESCAPES
line 6 “lib/parser/lexer-strings.rl” %
- ESCAPE_WHITESPACE
- LEX_STATES
- REGEXP_META_CHARACTERS
Attributes
_lex_actions[RW]
_lex_eof_trans[RW]
_lex_from_state_actions[RW]
_lex_index_offsets[RW]
_lex_indicies[RW]
_lex_key_spans[RW]
_lex_to_state_actions[RW]
_lex_trans_actions[RW]
_lex_trans_keys[RW]
_lex_trans_targs[RW]
lex_en_character[RW]
lex_en_interp_backslash_delimited[RW]
lex_en_interp_backslash_delimited_words[RW]
lex_en_interp_string[RW]
lex_en_interp_words[RW]
lex_en_plain_backslash_delimited[RW]
lex_en_plain_backslash_delimited_words[RW]
lex_en_plain_string[RW]
lex_en_plain_words[RW]
lex_en_regexp_modifiers[RW]
lex_en_unknown[RW]
lex_error[RW]
lex_start[RW]
herebody_s[RW]
source_buffer[RW]
Set by “main” lexer
source_pts[RW]
Set by “main” lexer
Public Class Methods
new(lexer, version)
click to toggle source
# File lib/parser/lexer-strings.rb, line 3299 def initialize(lexer, version) @lexer = lexer @version = version @_lex_actions = if self.class.respond_to?(:_lex_actions, true) self.class.send :_lex_actions else [] end reset end
Public Instance Methods
advance(p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 3338 def advance(p) # Ugly, but dependent on Ragel output. Consider refactoring it somehow. klass = self.class _lex_trans_keys = klass.send :_lex_trans_keys _lex_key_spans = klass.send :_lex_key_spans _lex_index_offsets = klass.send :_lex_index_offsets _lex_indicies = klass.send :_lex_indicies _lex_trans_targs = klass.send :_lex_trans_targs _lex_trans_actions = klass.send :_lex_trans_actions _lex_to_state_actions = klass.send :_lex_to_state_actions _lex_from_state_actions = klass.send :_lex_from_state_actions _lex_eof_trans = klass.send :_lex_eof_trans _lex_actions = @_lex_actions pe = source_pts.size + 2 eof = pe # line 3357 "lib/parser/lexer-strings.rb" begin # ragel flat testEof = false _slen, _trans, _keys, _inds, _acts, _nacts = nil _goto_level = 0 _resume = 10 _eof_trans = 15 _again = 20 _test_eof = 30 _out = 40 while true _trigger_goto = false if _goto_level <= 0 if p == pe _goto_level = _test_eof next end if @cs == 0 _goto_level = _out next end end if _goto_level <= _resume _acts = _lex_from_state_actions[ @cs] _nacts = _lex_actions[_acts] _acts += 1 while _nacts > 0 _nacts -= 1 _acts += 1 case _lex_actions[_acts - 1] when 24 then # line 1 "NONE" begin @ts = p end # line 3392 "lib/parser/lexer-strings.rb" end # from state action switch end if _trigger_goto next end _keys = @cs << 1 _inds = _lex_index_offsets[ @cs] _slen = _lex_key_spans[ @cs] _wide = ( (source_pts[p] || 0)) _trans = if ( _slen > 0 && _lex_trans_keys[_keys] <= _wide && _wide <= _lex_trans_keys[_keys + 1] ) then _lex_indicies[ _inds + _wide - _lex_trans_keys[_keys] ] else _lex_indicies[ _inds + _slen ] end end if _goto_level <= _eof_trans @cs = _lex_trans_targs[_trans] if _lex_trans_actions[_trans] != 0 _acts = _lex_trans_actions[_trans] _nacts = _lex_actions[_acts] _acts += 1 while _nacts > 0 _nacts -= 1 _acts += 1 case _lex_actions[_acts - 1] when 0 then # line 525 "lib/parser/lexer-strings.rl" begin # Record position of a newline for precise location reporting on tNL # tokens. # # This action is embedded directly into c_nl, as it is idempotent and # there are no cases when we need to skip it. @newline_s = p end when 1 then # line 581 "lib/parser/lexer-strings.rl" begin unicode_points(p) end when 2 then # line 585 "lib/parser/lexer-strings.rl" begin unescape_char(p) end when 3 then # line 589 "lib/parser/lexer-strings.rl" begin diagnostic :fatal, :invalid_escape end when 4 then # line 593 "lib/parser/lexer-strings.rl" begin read_post_meta_or_ctrl_char(p) end when 5 then # line 597 "lib/parser/lexer-strings.rl" begin slash_c_char end when 6 then # line 601 "lib/parser/lexer-strings.rl" begin slash_m_char end when 7 then # line 607 "lib/parser/lexer-strings.rl" begin encode_escaped_char(p) end when 8 then # line 613 "lib/parser/lexer-strings.rl" begin @escape = "\x7f" end when 9 then # line 614 "lib/parser/lexer-strings.rl" begin encode_escaped_char(p) end when 10 then # line 621 "lib/parser/lexer-strings.rl" begin @escape = encode_escape(tok(@escape_s, p).to_i(8) % 0x100) end when 11 then # line 625 "lib/parser/lexer-strings.rl" begin @escape = encode_escape(tok(@escape_s + 1, p).to_i(16)) end when 12 then # line 629 "lib/parser/lexer-strings.rl" begin diagnostic :fatal, :invalid_hex_escape, nil, range(@escape_s - 1, p + 2) end when 13 then # line 635 "lib/parser/lexer-strings.rl" begin @escape = tok(@escape_s + 1, p).to_i(16).chr(Encoding::UTF_8) end when 14 then # line 639 "lib/parser/lexer-strings.rl" begin check_invalid_escapes(p) end when 15 then # line 645 "lib/parser/lexer-strings.rl" begin check_invalid_escapes(p) end when 16 then # line 659 "lib/parser/lexer-strings.rl" begin diagnostic :fatal, :unterminated_unicode, nil, range(p - 1, p) end when 17 then # line 685 "lib/parser/lexer-strings.rl" begin diagnostic :fatal, :escape_eof, nil, range(p - 1, p) end when 18 then # line 691 "lib/parser/lexer-strings.rl" begin @escape_s = p @escape = nil end when 19 then # line 808 "lib/parser/lexer-strings.rl" begin interp_var_kind = :gvar end when 20 then # line 809 "lib/parser/lexer-strings.rl" begin interp_var_kind = :cvar end when 21 then # line 810 "lib/parser/lexer-strings.rl" begin interp_var_kind = :ivar end when 22 then # line 944 "lib/parser/lexer-strings.rl" begin @escape = nil end when 25 then # line 1 "NONE" begin @te = p+1 end when 26 then # line 842 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_interp_code(current_literal) @root_lexer_state = @lexer.class.lex_en_expr_value; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 27 then # line 761 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_string_eol_check_eof(current_literal, pe) if current_literal.heredoc? line = extend_string_eol_heredoc_line # Try ending the heredoc with the complete most recently # scanned line. @herebody_s always refers to the start of such line. if current_literal.nest_and_try_closing(line, @herebody_s, @ts) # Adjust @herebody_s to point to the next line. @herebody_s = @te # Continue regular lexing after the heredoc reference (<<END). p = current_literal.heredoc_e - 1 @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end else # Calculate indentation level for <<~HEREDOCs. current_literal.infer_indent_level(line) # Ditto. @herebody_s = @te end else # Try ending the literal with a newline. if current_literal.nest_and_try_closing(tok, @ts, @te) @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end end p = extend_string_eol_heredoc_intertwined(p) end extend_string_eol_words(current_literal, p) end end when 28 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p+1 begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 29 then # line 824 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin extend_interp_digit_var end end when 30 then # line 813 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin current_literal = literal extend_interp_var(current_literal) emit_interp_var(interp_var_kind) end end when 31 then # line 754 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin extend_string_escaped end end when 32 then # line 796 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin literal.extend_space @ts, @te end end when 33 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 34 then # line 754 "lib/parser/lexer-strings.rl" begin begin p = (( @te))-1; end begin extend_string_escaped end end when 35 then # line 731 "lib/parser/lexer-strings.rl" begin begin p = (( @te))-1; end begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 36 then # line 842 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_interp_code(current_literal) @root_lexer_state = @lexer.class.lex_en_expr_value; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 37 then # line 761 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_string_eol_check_eof(current_literal, pe) if current_literal.heredoc? line = extend_string_eol_heredoc_line # Try ending the heredoc with the complete most recently # scanned line. @herebody_s always refers to the start of such line. if current_literal.nest_and_try_closing(line, @herebody_s, @ts) # Adjust @herebody_s to point to the next line. @herebody_s = @te # Continue regular lexing after the heredoc reference (<<END). p = current_literal.heredoc_e - 1 @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end else # Calculate indentation level for <<~HEREDOCs. current_literal.infer_indent_level(line) # Ditto. @herebody_s = @te end else # Try ending the literal with a newline. if current_literal.nest_and_try_closing(tok, @ts, @te) @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end end p = extend_string_eol_heredoc_intertwined(p) end extend_string_eol_words(current_literal, p) end end when 38 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p+1 begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 39 then # line 824 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin extend_interp_digit_var end end when 40 then # line 813 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin current_literal = literal extend_interp_var(current_literal) emit_interp_var(interp_var_kind) end end when 41 then # line 754 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin extend_string_escaped end end when 42 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 43 then # line 754 "lib/parser/lexer-strings.rl" begin begin p = (( @te))-1; end begin extend_string_escaped end end when 44 then # line 731 "lib/parser/lexer-strings.rl" begin begin p = (( @te))-1; end begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 45 then # line 754 "lib/parser/lexer-strings.rl" begin @te = p+1 begin extend_string_escaped end end when 46 then # line 761 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_string_eol_check_eof(current_literal, pe) if current_literal.heredoc? line = extend_string_eol_heredoc_line # Try ending the heredoc with the complete most recently # scanned line. @herebody_s always refers to the start of such line. if current_literal.nest_and_try_closing(line, @herebody_s, @ts) # Adjust @herebody_s to point to the next line. @herebody_s = @te # Continue regular lexing after the heredoc reference (<<END). p = current_literal.heredoc_e - 1 @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end else # Calculate indentation level for <<~HEREDOCs. current_literal.infer_indent_level(line) # Ditto. @herebody_s = @te end else # Try ending the literal with a newline. if current_literal.nest_and_try_closing(tok, @ts, @te) @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end end p = extend_string_eol_heredoc_intertwined(p) end extend_string_eol_words(current_literal, p) end end when 47 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p+1 begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 48 then # line 796 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin literal.extend_space @ts, @te end end when 49 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 50 then # line 761 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_string_eol_check_eof(current_literal, pe) if current_literal.heredoc? line = extend_string_eol_heredoc_line # Try ending the heredoc with the complete most recently # scanned line. @herebody_s always refers to the start of such line. if current_literal.nest_and_try_closing(line, @herebody_s, @ts) # Adjust @herebody_s to point to the next line. @herebody_s = @te # Continue regular lexing after the heredoc reference (<<END). p = current_literal.heredoc_e - 1 @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end else # Calculate indentation level for <<~HEREDOCs. current_literal.infer_indent_level(line) # Ditto. @herebody_s = @te end else # Try ending the literal with a newline. if current_literal.nest_and_try_closing(tok, @ts, @te) @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end end p = extend_string_eol_heredoc_intertwined(p) end extend_string_eol_words(current_literal, p) end end when 51 then # line 754 "lib/parser/lexer-strings.rl" begin @te = p+1 begin extend_string_escaped end end when 52 then # line 761 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_string_eol_check_eof(current_literal, pe) if current_literal.heredoc? line = extend_string_eol_heredoc_line # Try ending the heredoc with the complete most recently # scanned line. @herebody_s always refers to the start of such line. if current_literal.nest_and_try_closing(line, @herebody_s, @ts) # Adjust @herebody_s to point to the next line. @herebody_s = @te # Continue regular lexing after the heredoc reference (<<END). p = current_literal.heredoc_e - 1 @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end else # Calculate indentation level for <<~HEREDOCs. current_literal.infer_indent_level(line) # Ditto. @herebody_s = @te end else # Try ending the literal with a newline. if current_literal.nest_and_try_closing(tok, @ts, @te) @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end end p = extend_string_eol_heredoc_intertwined(p) end extend_string_eol_words(current_literal, p) end end when 53 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p+1 begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 54 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 55 then # line 842 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_interp_code(current_literal) @root_lexer_state = @lexer.class.lex_en_expr_value; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 56 then # line 761 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_string_eol_check_eof(current_literal, pe) if current_literal.heredoc? line = extend_string_eol_heredoc_line # Try ending the heredoc with the complete most recently # scanned line. @herebody_s always refers to the start of such line. if current_literal.nest_and_try_closing(line, @herebody_s, @ts) # Adjust @herebody_s to point to the next line. @herebody_s = @te # Continue regular lexing after the heredoc reference (<<END). p = current_literal.heredoc_e - 1 @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end else # Calculate indentation level for <<~HEREDOCs. current_literal.infer_indent_level(line) # Ditto. @herebody_s = @te end else # Try ending the literal with a newline. if current_literal.nest_and_try_closing(tok, @ts, @te) @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end end p = extend_string_eol_heredoc_intertwined(p) end extend_string_eol_words(current_literal, p) end end when 57 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p+1 begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 58 then # line 824 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin extend_interp_digit_var end end when 59 then # line 813 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin current_literal = literal extend_interp_var(current_literal) emit_interp_var(interp_var_kind) end end when 60 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 61 then # line 731 "lib/parser/lexer-strings.rl" begin begin p = (( @te))-1; end begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 62 then # line 761 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_string_eol_check_eof(current_literal, pe) if current_literal.heredoc? line = extend_string_eol_heredoc_line # Try ending the heredoc with the complete most recently # scanned line. @herebody_s always refers to the start of such line. if current_literal.nest_and_try_closing(line, @herebody_s, @ts) # Adjust @herebody_s to point to the next line. @herebody_s = @te # Continue regular lexing after the heredoc reference (<<END). p = current_literal.heredoc_e - 1 @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end else # Calculate indentation level for <<~HEREDOCs. current_literal.infer_indent_level(line) # Ditto. @herebody_s = @te end else # Try ending the literal with a newline. if current_literal.nest_and_try_closing(tok, @ts, @te) @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end end p = extend_string_eol_heredoc_intertwined(p) end extend_string_eol_words(current_literal, p) end end when 63 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p+1 begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 64 then # line 842 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_interp_code(current_literal) @root_lexer_state = @lexer.class.lex_en_expr_value; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 65 then # line 761 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_string_eol_check_eof(current_literal, pe) if current_literal.heredoc? line = extend_string_eol_heredoc_line # Try ending the heredoc with the complete most recently # scanned line. @herebody_s always refers to the start of such line. if current_literal.nest_and_try_closing(line, @herebody_s, @ts) # Adjust @herebody_s to point to the next line. @herebody_s = @te # Continue regular lexing after the heredoc reference (<<END). p = current_literal.heredoc_e - 1 @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end else # Calculate indentation level for <<~HEREDOCs. current_literal.infer_indent_level(line) # Ditto. @herebody_s = @te end else # Try ending the literal with a newline. if current_literal.nest_and_try_closing(tok, @ts, @te) @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end end p = extend_string_eol_heredoc_intertwined(p) end extend_string_eol_words(current_literal, p) end end when 66 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p+1 begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 67 then # line 824 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin extend_interp_digit_var end end when 68 then # line 813 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin current_literal = literal extend_interp_var(current_literal) emit_interp_var(interp_var_kind) end end when 69 then # line 796 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin literal.extend_space @ts, @te end end when 70 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 71 then # line 731 "lib/parser/lexer-strings.rl" begin begin p = (( @te))-1; end begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 72 then # line 761 "lib/parser/lexer-strings.rl" begin @te = p+1 begin current_literal = literal extend_string_eol_check_eof(current_literal, pe) if current_literal.heredoc? line = extend_string_eol_heredoc_line # Try ending the heredoc with the complete most recently # scanned line. @herebody_s always refers to the start of such line. if current_literal.nest_and_try_closing(line, @herebody_s, @ts) # Adjust @herebody_s to point to the next line. @herebody_s = @te # Continue regular lexing after the heredoc reference (<<END). p = current_literal.heredoc_e - 1 @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end else # Calculate indentation level for <<~HEREDOCs. current_literal.infer_indent_level(line) # Ditto. @herebody_s = @te end else # Try ending the literal with a newline. if current_literal.nest_and_try_closing(tok, @ts, @te) @cs = (pop_literal); begin p += 1 _trigger_goto = true _goto_level = _out break end end p = extend_string_eol_heredoc_intertwined(p) end extend_string_eol_words(current_literal, p) end end when 73 then # line 731 "lib/parser/lexer-strings.rl" begin @te = p+1 begin string = tok lookahead = extend_string_slice_end(lookahead) current_literal = literal if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead)) if token[0] == :tLABEL_END p += 1 pop_literal @root_lexer_state = @lexer.class.lex_en_expr_labelarg else if state = pop_literal @cs = (state); end end begin p += 1 _trigger_goto = true _goto_level = _out break end else extend_string_for_token_range(current_literal, string) end end end when 74 then # line 796 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin literal.extend_space @ts, @te end end when 75 then # line 928 "lib/parser/lexer-strings.rl" begin @te = p+1 begin emit(:tREGEXP_OPT, tok(@ts, @te - 1), @ts, @te - 1) p = p - 1; @root_lexer_state = @lexer.class.lex_en_expr_end; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 76 then # line 915 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin unknown_options = tok.scan(/[^imxouesn]/) if unknown_options.any? diagnostic :error, :regexp_options, { :options => unknown_options.join } end emit(:tREGEXP_OPT) @root_lexer_state = @lexer.class.lex_en_expr_end; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 77 then # line 953 "lib/parser/lexer-strings.rl" begin @te = p+1 begin escape = ESCAPE_WHITESPACE[source_buffer.slice(@ts + 1, 1)] diagnostic :warning, :invalid_escape_use, { :escape => escape }, range p = @ts - 1 @root_lexer_state = @lexer.class.lex_en_expr_end; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 78 then # line 946 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin emit_character_constant @root_lexer_state = @lexer.class.lex_en_expr_end; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 79 then # line 964 "lib/parser/lexer-strings.rl" begin @te = p p = p - 1; begin p = @ts - 1 @root_lexer_state = @lexer.class.lex_en_expr_end; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 80 then # line 946 "lib/parser/lexer-strings.rl" begin begin p = (( @te))-1; end begin emit_character_constant @root_lexer_state = @lexer.class.lex_en_expr_end; begin p += 1 _trigger_goto = true _goto_level = _out break end end end when 81 then # line 972 "lib/parser/lexer-strings.rl" begin @te = p+1 begin raise 'bug' end end # line 4935 "lib/parser/lexer-strings.rb" end # action switch end end if _trigger_goto next end end if _goto_level <= _again _acts = _lex_to_state_actions[ @cs] _nacts = _lex_actions[_acts] _acts += 1 while _nacts > 0 _nacts -= 1 _acts += 1 case _lex_actions[_acts - 1] when 23 then # line 1 "NONE" begin @ts = nil; end # line 4955 "lib/parser/lexer-strings.rb" end # to state action switch end if _trigger_goto next end if @cs == 0 _goto_level = _out next end p += 1 if p != pe _goto_level = _resume next end end if _goto_level <= _test_eof if p == eof if _lex_eof_trans[ @cs] > 0 _trans = _lex_eof_trans[ @cs] - 1; _goto_level = _eof_trans next; end end end if _goto_level <= _out break end end end # line 78 "lib/parser/lexer-strings.rl" # % # Ragel creates a local variable called `testEof` but it doesn't use # it in any assignment. This dead code is here to swallow the warning. # It has no runtime cost because Ruby doesn't produce any instructions from it. if false testEof end [p, @root_lexer_state] end
close_interp_on_current_literal(p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5068 def close_interp_on_current_literal(p) current_literal = literal if current_literal if current_literal.end_interp_brace_and_try_closing if version?(18, 19) emit(:tRCURLY, '}'.freeze, p - 1, p) @lexer.cond.lexpop @lexer.cmdarg.lexpop else emit(:tSTRING_DEND, '}'.freeze, p - 1, p) end if current_literal.saved_herebody_s @herebody_s = current_literal.saved_herebody_s end continue_lexing(current_literal) return true end end end
continue_lexing(current_literal)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5042 def continue_lexing(current_literal) @cs = next_state_for_literal(current_literal) end
dedent_level()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5091 def dedent_level # We erase @dedent_level as a precaution to avoid accidentally # using a stale value. dedent_level, @dedent_level = @dedent_level, nil dedent_level end
literal()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5046 def literal @literal_stack.last end
next_state_for_literal(literal)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5014 def next_state_for_literal(literal) if literal.words? && literal.backslash_delimited? if literal.interpolate? self.class.lex_en_interp_backslash_delimited_words else self.class.lex_en_plain_backslash_delimited_words end elsif literal.words? && !literal.backslash_delimited? if literal.interpolate? self.class.lex_en_interp_words else self.class.lex_en_plain_words end elsif !literal.words? && literal.backslash_delimited? if literal.interpolate? self.class.lex_en_interp_backslash_delimited else self.class.lex_en_plain_backslash_delimited end else if literal.interpolate? self.class.lex_en_interp_string else self.class.lex_en_plain_string end end end
on_newline(p)
click to toggle source
This hook is triggered by “main” lexer on every newline character
# File lib/parser/lexer-strings.rb, line 5099 def on_newline(p) # After every heredoc was parsed, @herebody_s contains the # position of next token after all heredocs. if @herebody_s p = @herebody_s @herebody_s = nil end p end
pop_literal()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5050 def pop_literal old_literal = @literal_stack.pop @dedent_level = old_literal.dedent_level if old_literal.type == :tREGEXP_BEG @root_lexer_state = @lexer.class.lex_en_inside_string # Fetch modifiers. self.class.lex_en_regexp_modifiers else @root_lexer_state = @lexer.class.lex_en_expr_end # Do nothing, yield to main lexer nil end end
push_literal(*args)
click to toggle source
read_character_constant(p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 4998 def read_character_constant(p) @cs = self.class.lex_en_character advance(p) end
reset()
click to toggle source
# File lib/parser/lexer-strings.rb, line 3313 def reset @cs = self.class.lex_en_unknown @literal_stack = [] @escape_s = nil # starting position of current sequence @escape = nil # last escaped sequence, as string @herebody_s = nil # starting position of current heredoc line # After encountering the closing line of <<~SQUIGGLY_HEREDOC, # we store the indentation level and give it out to the parser # on request. It is not possible to infer indentation level just # from the AST because escape sequences such as `\ ` or `\t` are # expanded inside the lexer, but count as non-whitespace for # indentation purposes. @dedent_level = nil end
Protected Instance Methods
check_ambiguous_slash(tm)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5396 def check_ambiguous_slash(tm) if tok(tm, tm + 1) == '/'.freeze # Ambiguous regexp literal. if @version < 30 diagnostic :warning, :ambiguous_literal, nil, range(tm, tm + 1) else diagnostic :warning, :ambiguous_regexp, nil, range(tm, tm + 1) end end end
check_invalid_escapes(p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5407 def check_invalid_escapes(p) if emit_invalid_escapes? diagnostic :fatal, :invalid_unicode_escape, nil, range(@escape_s - 1, p) end end
cond()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5135 def cond @lexer.cond end
diagnostic(type, reason, arguments=nil, location=range, highlights=[])
click to toggle source
# File lib/parser/lexer-strings.rb, line 5131 def diagnostic(type, reason, arguments=nil, location=range, highlights=[]) @lexer.send(:diagnostic, type, reason, arguments, location, highlights) end
emit(type, value = tok, s = @ts, e = @te)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5127 def emit(type, value = tok, s = @ts, e = @te) @lexer.send(:emit, type, value, s, e) end
emit_character_constant()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5386 def emit_character_constant value = @escape || tok(@ts + 1) if version?(18) emit(:tINTEGER, value.getbyte(0)) else emit(:tCHARACTER, value) end end
emit_interp_var(interp_var_kind)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5363 def emit_interp_var(interp_var_kind) case interp_var_kind when :cvar @lexer.send(:emit_class_var, @ts + 1, @te) when :ivar @lexer.send(:emit_instance_var, @ts + 1, @te) when :gvar @lexer.send(:emit_global_var, @ts + 1, @te) end end
emit_invalid_escapes?()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5139 def emit_invalid_escapes? # always true for old Rubies return true if @version < 32 # in "?\u123" case we don't push any literals # but we always emit invalid escapes return true if literal.nil? # Ruby >= 32, regexp, exceptional case !literal.regexp? end
encode_escape(ord)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5290 def encode_escape(ord) ord.chr.force_encoding(source_buffer.source.encoding) end
encode_escaped_char(p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5374 def encode_escaped_char(p) @escape = encode_escape(tok(p - 2, p).to_i(16)) end
eof_codepoint?(point)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5111 def eof_codepoint?(point) [0x04, 0x1a, 0x00].include? point end
extend_interp_code(current_literal)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5209 def extend_interp_code(current_literal) current_literal.flush_string current_literal.extend_content emit(:tSTRING_DBEG, '#{'.freeze) if current_literal.heredoc? current_literal.saved_herebody_s = @herebody_s @herebody_s = nil end current_literal.start_interp_brace @lexer.command_start = true end
extend_interp_digit_var()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5224 def extend_interp_digit_var if @version >= 27 literal.extend_string(tok, @ts, @te) else message = tok.start_with?('#@@') ? :cvar_name : :ivar_name diagnostic :error, message, { :name => tok(@ts + 1, @te) }, range(@ts + 1, @te) end end
extend_interp_var(current_literal)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5354 def extend_interp_var(current_literal) current_literal.flush_string current_literal.extend_content emit(:tSTRING_DVAR, nil, @ts, @ts + 1) @ts end
extend_string_eol_check_eof(current_literal, pe)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5233 def extend_string_eol_check_eof(current_literal, pe) if @te == pe diagnostic :fatal, :string_eof, nil, range(current_literal.str_s, current_literal.str_s + 1) end end
extend_string_eol_heredoc_intertwined(p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5250 def extend_string_eol_heredoc_intertwined(p) if @herebody_s # This is a regular literal intertwined with a heredoc. Like: # # p <<-foo+"1 # bar # foo # 2" # # which, incidentally, evaluates to "bar\n1\n2". p = @herebody_s - 1 @herebody_s = nil end p end
extend_string_eol_heredoc_line()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5240 def extend_string_eol_heredoc_line line = tok(@herebody_s, @ts).gsub(/\r+$/, ''.freeze) if version?(18, 19, 20) # See ruby:c48b4209c line = line.gsub(/\r.*$/, ''.freeze) end line end
extend_string_eol_words(current_literal, p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5266 def extend_string_eol_words(current_literal, p) if current_literal.words? && !eof_codepoint?(source_pts[p]) current_literal.extend_space @ts, @te else # A literal newline is appended if the heredoc was _not_ closed # this time (see fbreak above). See also Literal#nest_and_try_closing # for rationale of calling #flush_string here. current_literal.extend_string tok, @ts, @te current_literal.flush_string end end
extend_string_escaped()
click to toggle source
String escaping
# File lib/parser/lexer-strings.rb, line 5153 def extend_string_escaped current_literal = literal # Get the first character after the backslash. escaped_char = source_buffer.slice(@escape_s, 1).chr if current_literal.munge_escape? escaped_char # If this particular literal uses this character as an opening # or closing delimiter, it is an escape sequence for that # particular character. Write it without the backslash. if current_literal.regexp? && REGEXP_META_CHARACTERS.match(escaped_char) # Regular expressions should include escaped delimiters in their # escaped form, except when the escaped character is # a closing delimiter but not a regexp metacharacter. # # The backslash itself cannot be used as a closing delimiter # at the same time as an escape symbol, but it is always munged, # so this branch also executes for the non-closing-delimiter case # for the backslash. current_literal.extend_string(tok, @ts, @te) else current_literal.extend_string(escaped_char, @ts, @te) end else # It does not. So this is an actual escape sequence, yay! if current_literal.squiggly_heredoc? && escaped_char == "\n".freeze # Squiggly heredocs like # <<~-HERE # 1\ # 2 # HERE # treat '\' as a line continuation, but still dedent the body, so the heredoc above becomes "12\n". # This information is emitted as is, without escaping, # later this escape sequence (\\\n) gets handled manually in the Lexer::Dedenter current_literal.extend_string(tok, @ts, @te) elsif current_literal.supports_line_continuation_via_slash? && escaped_char == "\n".freeze # Heredocs, regexp and a few other types of literals support line # continuation via \\\n sequence. The code like # "a\ # b" # must be parsed as "ab" current_literal.extend_string(tok.gsub("\\\n".freeze, ''.freeze), @ts, @te) elsif current_literal.regexp? && @version >= 31 && %w[c C m M].include?(escaped_char) # Ruby >= 3.1 escapes \c- and \m chars, that's the only escape sequence # supported by regexes so far, so it needs a separate branch. current_literal.extend_string(@escape, @ts, @te) elsif current_literal.regexp? # Regular expressions should include escape sequences in their # escaped form. On the other hand, escaped newlines are removed (in cases like "\\C-\\\n\\M-x") current_literal.extend_string(tok.gsub("\\\n".freeze, ''.freeze), @ts, @te) else current_literal.extend_string(@escape || tok, @ts, @te) end end end
extend_string_for_token_range(current_literal, string)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5286 def extend_string_for_token_range(current_literal, string) current_literal.extend_string(string, @ts, @te) end
extend_string_slice_end(lookahead)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5278 def extend_string_slice_end(lookahead) # tLABEL_END is only possible in non-cond context on >= 2.2 if @version >= 22 && !cond.active? lookahead = source_buffer.slice(@te, 2) end lookahead end
range(s = @ts, e = @te)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5123 def range(s = @ts, e = @te) Parser::Source::Range.new(@source_buffer, s, e) end
read_post_meta_or_ctrl_char(p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5346 def read_post_meta_or_ctrl_char(p) @escape = source_buffer.slice(p - 1, 1).chr if @version >= 27 && ((0..8).include?(@escape.ord) || (14..31).include?(@escape.ord)) diagnostic :fatal, :invalid_escape end end
slash_c_char()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5378 def slash_c_char @escape = encode_escape(@escape[0].ord & 0x9f) end
slash_m_char()
click to toggle source
# File lib/parser/lexer-strings.rb, line 5382 def slash_m_char @escape = encode_escape(@escape[0].ord | 0x80) end
tok(s = @ts, e = @te)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5119 def tok(s = @ts, e = @te) @source_buffer.slice(s, e - s) end
unescape_char(p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5294 def unescape_char(p) codepoint = source_pts[p - 1] if @version >= 30 && (codepoint == 117 || codepoint == 85) # 'u' or 'U' diagnostic :fatal, :invalid_escape end if (@escape = ESCAPES[codepoint]).nil? @escape = encode_escape(source_buffer.slice(p - 1, 1)) end end
unicode_points(p)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5306 def unicode_points(p) @escape = "" codepoints = tok(@escape_s + 2, p - 1) codepoint_s = @escape_s + 2 if @version < 24 if codepoints.start_with?(" ") || codepoints.start_with?("\t") diagnostic :fatal, :invalid_unicode_escape, nil, range(@escape_s + 2, @escape_s + 3) end if spaces_p = codepoints.index(/[ \t]{2}/) diagnostic :fatal, :invalid_unicode_escape, nil, range(codepoint_s + spaces_p + 1, codepoint_s + spaces_p + 2) end if codepoints.end_with?(" ") || codepoints.end_with?("\t") diagnostic :fatal, :invalid_unicode_escape, nil, range(p - 1, p) end end codepoints.scan(/([0-9a-fA-F]+)|([ \t]+)/).each do |(codepoint_str, spaces)| if spaces codepoint_s += spaces.length else codepoint = codepoint_str.to_i(16) if codepoint >= 0x110000 diagnostic :error, :unicode_point_too_large, nil, range(codepoint_s, codepoint_s + codepoint_str.length) break end @escape += codepoint.chr(Encoding::UTF_8) codepoint_s += codepoint_str.length end end end
version?(*versions)
click to toggle source
# File lib/parser/lexer-strings.rb, line 5115 def version?(*versions) versions.include?(@version) end