class Parser::LexerStrings

line 3 “lib/parser/lexer-strings.rl”

Constants

ESCAPES

line 6 “lib/parser/lexer-strings.rl” %

ESCAPE_WHITESPACE
LEX_STATES
REGEXP_META_CHARACTERS

Attributes

_lex_actions[RW]
_lex_eof_trans[RW]
_lex_from_state_actions[RW]
_lex_index_offsets[RW]
_lex_indicies[RW]
_lex_key_spans[RW]
_lex_to_state_actions[RW]
_lex_trans_actions[RW]
_lex_trans_keys[RW]
_lex_trans_targs[RW]
lex_en_character[RW]
lex_en_interp_backslash_delimited[RW]
lex_en_interp_backslash_delimited_words[RW]
lex_en_interp_string[RW]
lex_en_interp_words[RW]
lex_en_plain_backslash_delimited[RW]
lex_en_plain_backslash_delimited_words[RW]
lex_en_plain_string[RW]
lex_en_plain_words[RW]
lex_en_regexp_modifiers[RW]
lex_en_unknown[RW]
lex_error[RW]
lex_start[RW]
herebody_s[RW]
source_buffer[RW]

Set by “main” lexer

source_pts[RW]

Set by “main” lexer

Public Class Methods

new(lexer, version) click to toggle source
# File lib/parser/lexer-strings.rb, line 3299
def initialize(lexer, version)
  @lexer = lexer
  @version = version

  @_lex_actions =
    if self.class.respond_to?(:_lex_actions, true)
      self.class.send :_lex_actions
    else
      []
    end

  reset
end

Public Instance Methods

advance(p) click to toggle source
# File lib/parser/lexer-strings.rb, line 3338
  def advance(p)
    # Ugly, but dependent on Ragel output. Consider refactoring it somehow.
    klass = self.class
    _lex_trans_keys         = klass.send :_lex_trans_keys
    _lex_key_spans          = klass.send :_lex_key_spans
    _lex_index_offsets      = klass.send :_lex_index_offsets
    _lex_indicies           = klass.send :_lex_indicies
    _lex_trans_targs        = klass.send :_lex_trans_targs
    _lex_trans_actions      = klass.send :_lex_trans_actions
    _lex_to_state_actions   = klass.send :_lex_to_state_actions
    _lex_from_state_actions = klass.send :_lex_from_state_actions
    _lex_eof_trans          = klass.send :_lex_eof_trans
    _lex_actions            = @_lex_actions

    pe = source_pts.size + 2
    eof = pe

    
# line 3357 "lib/parser/lexer-strings.rb"
begin # ragel flat
        testEof = false
        _slen, _trans, _keys, _inds, _acts, _nacts = nil
        _goto_level = 0
        _resume = 10
        _eof_trans = 15
        _again = 20
        _test_eof = 30
        _out = 40
        while true
        _trigger_goto = false
        if _goto_level <= 0
        if p == pe
                _goto_level = _test_eof
                next
        end
        if  @cs == 0
                _goto_level = _out
                next
        end
        end
        if _goto_level <= _resume
        _acts = _lex_from_state_actions[ @cs]
        _nacts = _lex_actions[_acts]
        _acts += 1
        while _nacts > 0
                _nacts -= 1
                _acts += 1
                case _lex_actions[_acts - 1]
        when 24 then
# line 1 "NONE"
                begin
 @ts = p
                end
# line 3392 "lib/parser/lexer-strings.rb"
                end # from state action switch
        end
        if _trigger_goto
                next
        end
        _keys =  @cs << 1
        _inds = _lex_index_offsets[ @cs]
        _slen = _lex_key_spans[ @cs]
        _wide = ( (source_pts[p] || 0))
        _trans = if (   _slen > 0 && 
                        _lex_trans_keys[_keys] <= _wide && 
                        _wide <= _lex_trans_keys[_keys + 1] 
                    ) then
                        _lex_indicies[ _inds + _wide - _lex_trans_keys[_keys] ] 
                 else 
                        _lex_indicies[ _inds + _slen ]
                 end
        end
        if _goto_level <= _eof_trans
         @cs = _lex_trans_targs[_trans]
        if _lex_trans_actions[_trans] != 0
                _acts = _lex_trans_actions[_trans]
                _nacts = _lex_actions[_acts]
                _acts += 1
                while _nacts > 0
                        _nacts -= 1
                        _acts += 1
                        case _lex_actions[_acts - 1]
        when 0 then
# line 525 "lib/parser/lexer-strings.rl"
                begin

    # Record position of a newline for precise location reporting on tNL
    # tokens.
    #
    # This action is embedded directly into c_nl, as it is idempotent and
    # there are no cases when we need to skip it.
    @newline_s = p
                end
        when 1 then
# line 581 "lib/parser/lexer-strings.rl"
                begin

    unicode_points(p)
                end
        when 2 then
# line 585 "lib/parser/lexer-strings.rl"
                begin

    unescape_char(p)
                end
        when 3 then
# line 589 "lib/parser/lexer-strings.rl"
                begin

    diagnostic :fatal, :invalid_escape
                end
        when 4 then
# line 593 "lib/parser/lexer-strings.rl"
                begin

    read_post_meta_or_ctrl_char(p)
                end
        when 5 then
# line 597 "lib/parser/lexer-strings.rl"
                begin

    slash_c_char
                end
        when 6 then
# line 601 "lib/parser/lexer-strings.rl"
                begin

    slash_m_char
                end
        when 7 then
# line 607 "lib/parser/lexer-strings.rl"
                begin
 encode_escaped_char(p)                 end
        when 8 then
# line 613 "lib/parser/lexer-strings.rl"
                begin
 @escape = "\x7f"               end
        when 9 then
# line 614 "lib/parser/lexer-strings.rl"
                begin
 encode_escaped_char(p)                 end
        when 10 then
# line 621 "lib/parser/lexer-strings.rl"
                begin
 @escape = encode_escape(tok(@escape_s, p).to_i(8) % 0x100)             end
        when 11 then
# line 625 "lib/parser/lexer-strings.rl"
                begin
 @escape = encode_escape(tok(@escape_s + 1, p).to_i(16))                end
        when 12 then
# line 629 "lib/parser/lexer-strings.rl"
                begin

        diagnostic :fatal, :invalid_hex_escape, nil, range(@escape_s - 1, p + 2)
                end
        when 13 then
# line 635 "lib/parser/lexer-strings.rl"
                begin
 @escape = tok(@escape_s + 1, p).to_i(16).chr(Encoding::UTF_8)          end
        when 14 then
# line 639 "lib/parser/lexer-strings.rl"
                begin

        check_invalid_escapes(p)
                end
        when 15 then
# line 645 "lib/parser/lexer-strings.rl"
                begin

        check_invalid_escapes(p)
                end
        when 16 then
# line 659 "lib/parser/lexer-strings.rl"
                begin

          diagnostic :fatal, :unterminated_unicode, nil, range(p - 1, p)
                        end
        when 17 then
# line 685 "lib/parser/lexer-strings.rl"
                begin

      diagnostic :fatal, :escape_eof, nil, range(p - 1, p)
                end
        when 18 then
# line 691 "lib/parser/lexer-strings.rl"
                begin

    @escape_s = p
    @escape   = nil
                end
        when 19 then
# line 808 "lib/parser/lexer-strings.rl"
                begin
 interp_var_kind = :gvar                end
        when 20 then
# line 809 "lib/parser/lexer-strings.rl"
                begin
 interp_var_kind = :cvar                end
        when 21 then
# line 810 "lib/parser/lexer-strings.rl"
                begin
 interp_var_kind = :ivar                end
        when 22 then
# line 944 "lib/parser/lexer-strings.rl"
                begin
 @escape = nil          end
        when 25 then
# line 1 "NONE"
                begin
 @te = p+1
                end
        when 26 then
# line 842 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_interp_code(current_literal)
    @root_lexer_state = @lexer.class.lex_en_expr_value;
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

   end
                end
        when 27 then
# line 761 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_string_eol_check_eof(current_literal, pe)

    if current_literal.heredoc?
      line = extend_string_eol_heredoc_line

      # Try ending the heredoc with the complete most recently
      # scanned line. @herebody_s always refers to the start of such line.
      if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
        # Adjust @herebody_s to point to the next line.
        @herebody_s = @te

        # Continue regular lexing after the heredoc reference (<<END).
        p = current_literal.heredoc_e - 1
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      else
        # Calculate indentation level for <<~HEREDOCs.
        current_literal.infer_indent_level(line)

        # Ditto.
        @herebody_s = @te
      end
    else
      # Try ending the literal with a newline.
      if current_literal.nest_and_try_closing(tok, @ts, @te)
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      end

      p = extend_string_eol_heredoc_intertwined(p)
    end

    extend_string_eol_words(current_literal, p)
   end
                end
        when 28 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 29 then
# line 824 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    extend_interp_digit_var
   end
                end
        when 30 then
# line 813 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    current_literal = literal
    extend_interp_var(current_literal)
    emit_interp_var(interp_var_kind)
   end
                end
        when 31 then
# line 754 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    extend_string_escaped
   end
                end
        when 32 then
# line 796 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    literal.extend_space @ts, @te
   end
                end
        when 33 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 34 then
# line 754 "lib/parser/lexer-strings.rl"
                begin
 begin p = (( @te))-1; end
 begin 
    extend_string_escaped
   end
                end
        when 35 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 begin p = (( @te))-1; end
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 36 then
# line 842 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_interp_code(current_literal)
    @root_lexer_state = @lexer.class.lex_en_expr_value;
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

   end
                end
        when 37 then
# line 761 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_string_eol_check_eof(current_literal, pe)

    if current_literal.heredoc?
      line = extend_string_eol_heredoc_line

      # Try ending the heredoc with the complete most recently
      # scanned line. @herebody_s always refers to the start of such line.
      if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
        # Adjust @herebody_s to point to the next line.
        @herebody_s = @te

        # Continue regular lexing after the heredoc reference (<<END).
        p = current_literal.heredoc_e - 1
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      else
        # Calculate indentation level for <<~HEREDOCs.
        current_literal.infer_indent_level(line)

        # Ditto.
        @herebody_s = @te
      end
    else
      # Try ending the literal with a newline.
      if current_literal.nest_and_try_closing(tok, @ts, @te)
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      end

      p = extend_string_eol_heredoc_intertwined(p)
    end

    extend_string_eol_words(current_literal, p)
   end
                end
        when 38 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 39 then
# line 824 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    extend_interp_digit_var
   end
                end
        when 40 then
# line 813 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    current_literal = literal
    extend_interp_var(current_literal)
    emit_interp_var(interp_var_kind)
   end
                end
        when 41 then
# line 754 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    extend_string_escaped
   end
                end
        when 42 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 43 then
# line 754 "lib/parser/lexer-strings.rl"
                begin
 begin p = (( @te))-1; end
 begin 
    extend_string_escaped
   end
                end
        when 44 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 begin p = (( @te))-1; end
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 45 then
# line 754 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    extend_string_escaped
   end
                end
        when 46 then
# line 761 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_string_eol_check_eof(current_literal, pe)

    if current_literal.heredoc?
      line = extend_string_eol_heredoc_line

      # Try ending the heredoc with the complete most recently
      # scanned line. @herebody_s always refers to the start of such line.
      if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
        # Adjust @herebody_s to point to the next line.
        @herebody_s = @te

        # Continue regular lexing after the heredoc reference (<<END).
        p = current_literal.heredoc_e - 1
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      else
        # Calculate indentation level for <<~HEREDOCs.
        current_literal.infer_indent_level(line)

        # Ditto.
        @herebody_s = @te
      end
    else
      # Try ending the literal with a newline.
      if current_literal.nest_and_try_closing(tok, @ts, @te)
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      end

      p = extend_string_eol_heredoc_intertwined(p)
    end

    extend_string_eol_words(current_literal, p)
   end
                end
        when 47 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 48 then
# line 796 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    literal.extend_space @ts, @te
   end
                end
        when 49 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 50 then
# line 761 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_string_eol_check_eof(current_literal, pe)

    if current_literal.heredoc?
      line = extend_string_eol_heredoc_line

      # Try ending the heredoc with the complete most recently
      # scanned line. @herebody_s always refers to the start of such line.
      if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
        # Adjust @herebody_s to point to the next line.
        @herebody_s = @te

        # Continue regular lexing after the heredoc reference (<<END).
        p = current_literal.heredoc_e - 1
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      else
        # Calculate indentation level for <<~HEREDOCs.
        current_literal.infer_indent_level(line)

        # Ditto.
        @herebody_s = @te
      end
    else
      # Try ending the literal with a newline.
      if current_literal.nest_and_try_closing(tok, @ts, @te)
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      end

      p = extend_string_eol_heredoc_intertwined(p)
    end

    extend_string_eol_words(current_literal, p)
   end
                end
        when 51 then
# line 754 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    extend_string_escaped
   end
                end
        when 52 then
# line 761 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_string_eol_check_eof(current_literal, pe)

    if current_literal.heredoc?
      line = extend_string_eol_heredoc_line

      # Try ending the heredoc with the complete most recently
      # scanned line. @herebody_s always refers to the start of such line.
      if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
        # Adjust @herebody_s to point to the next line.
        @herebody_s = @te

        # Continue regular lexing after the heredoc reference (<<END).
        p = current_literal.heredoc_e - 1
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      else
        # Calculate indentation level for <<~HEREDOCs.
        current_literal.infer_indent_level(line)

        # Ditto.
        @herebody_s = @te
      end
    else
      # Try ending the literal with a newline.
      if current_literal.nest_and_try_closing(tok, @ts, @te)
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      end

      p = extend_string_eol_heredoc_intertwined(p)
    end

    extend_string_eol_words(current_literal, p)
   end
                end
        when 53 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 54 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 55 then
# line 842 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_interp_code(current_literal)
    @root_lexer_state = @lexer.class.lex_en_expr_value;
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

   end
                end
        when 56 then
# line 761 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_string_eol_check_eof(current_literal, pe)

    if current_literal.heredoc?
      line = extend_string_eol_heredoc_line

      # Try ending the heredoc with the complete most recently
      # scanned line. @herebody_s always refers to the start of such line.
      if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
        # Adjust @herebody_s to point to the next line.
        @herebody_s = @te

        # Continue regular lexing after the heredoc reference (<<END).
        p = current_literal.heredoc_e - 1
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      else
        # Calculate indentation level for <<~HEREDOCs.
        current_literal.infer_indent_level(line)

        # Ditto.
        @herebody_s = @te
      end
    else
      # Try ending the literal with a newline.
      if current_literal.nest_and_try_closing(tok, @ts, @te)
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      end

      p = extend_string_eol_heredoc_intertwined(p)
    end

    extend_string_eol_words(current_literal, p)
   end
                end
        when 57 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 58 then
# line 824 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    extend_interp_digit_var
   end
                end
        when 59 then
# line 813 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    current_literal = literal
    extend_interp_var(current_literal)
    emit_interp_var(interp_var_kind)
   end
                end
        when 60 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 61 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 begin p = (( @te))-1; end
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 62 then
# line 761 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_string_eol_check_eof(current_literal, pe)

    if current_literal.heredoc?
      line = extend_string_eol_heredoc_line

      # Try ending the heredoc with the complete most recently
      # scanned line. @herebody_s always refers to the start of such line.
      if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
        # Adjust @herebody_s to point to the next line.
        @herebody_s = @te

        # Continue regular lexing after the heredoc reference (<<END).
        p = current_literal.heredoc_e - 1
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      else
        # Calculate indentation level for <<~HEREDOCs.
        current_literal.infer_indent_level(line)

        # Ditto.
        @herebody_s = @te
      end
    else
      # Try ending the literal with a newline.
      if current_literal.nest_and_try_closing(tok, @ts, @te)
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      end

      p = extend_string_eol_heredoc_intertwined(p)
    end

    extend_string_eol_words(current_literal, p)
   end
                end
        when 63 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 64 then
# line 842 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_interp_code(current_literal)
    @root_lexer_state = @lexer.class.lex_en_expr_value;
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

   end
                end
        when 65 then
# line 761 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_string_eol_check_eof(current_literal, pe)

    if current_literal.heredoc?
      line = extend_string_eol_heredoc_line

      # Try ending the heredoc with the complete most recently
      # scanned line. @herebody_s always refers to the start of such line.
      if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
        # Adjust @herebody_s to point to the next line.
        @herebody_s = @te

        # Continue regular lexing after the heredoc reference (<<END).
        p = current_literal.heredoc_e - 1
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      else
        # Calculate indentation level for <<~HEREDOCs.
        current_literal.infer_indent_level(line)

        # Ditto.
        @herebody_s = @te
      end
    else
      # Try ending the literal with a newline.
      if current_literal.nest_and_try_closing(tok, @ts, @te)
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      end

      p = extend_string_eol_heredoc_intertwined(p)
    end

    extend_string_eol_words(current_literal, p)
   end
                end
        when 66 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 67 then
# line 824 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    extend_interp_digit_var
   end
                end
        when 68 then
# line 813 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    current_literal = literal
    extend_interp_var(current_literal)
    emit_interp_var(interp_var_kind)
   end
                end
        when 69 then
# line 796 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    literal.extend_space @ts, @te
   end
                end
        when 70 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 71 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 begin p = (( @te))-1; end
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 72 then
# line 761 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    current_literal = literal
    extend_string_eol_check_eof(current_literal, pe)

    if current_literal.heredoc?
      line = extend_string_eol_heredoc_line

      # Try ending the heredoc with the complete most recently
      # scanned line. @herebody_s always refers to the start of such line.
      if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
        # Adjust @herebody_s to point to the next line.
        @herebody_s = @te

        # Continue regular lexing after the heredoc reference (<<END).
        p = current_literal.heredoc_e - 1
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      else
        # Calculate indentation level for <<~HEREDOCs.
        current_literal.infer_indent_level(line)

        # Ditto.
        @herebody_s = @te
      end
    else
      # Try ending the literal with a newline.
      if current_literal.nest_and_try_closing(tok, @ts, @te)
         @cs = (pop_literal);   begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

      end

      p = extend_string_eol_heredoc_intertwined(p)
    end

    extend_string_eol_words(current_literal, p)
   end
                end
        when 73 then
# line 731 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
    string = tok

    lookahead = extend_string_slice_end(lookahead)

    current_literal = literal
    if !current_literal.heredoc? &&
          (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
      if token[0] == :tLABEL_END
        p += 1
        pop_literal
        @root_lexer_state = @lexer.class.lex_en_expr_labelarg
      else
        if state = pop_literal
           @cs = (state);
        end
      end
        begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

    else
      extend_string_for_token_range(current_literal, string)
    end
   end
                end
        when 74 then
# line 796 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
    literal.extend_space @ts, @te
   end
                end
        when 75 then
# line 928 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
        emit(:tREGEXP_OPT, tok(@ts, @te - 1), @ts, @te - 1)
        p = p - 1;
        @root_lexer_state = @lexer.class.lex_en_expr_end;
                begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

       end
                end
        when 76 then
# line 915 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
        unknown_options = tok.scan(/[^imxouesn]/)
        if unknown_options.any?
          diagnostic :error, :regexp_options,
                     { :options => unknown_options.join }
        end

        emit(:tREGEXP_OPT)
        @root_lexer_state = @lexer.class.lex_en_expr_end;
                begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

       end
                end
        when 77 then
# line 953 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin 
        escape = ESCAPE_WHITESPACE[source_buffer.slice(@ts + 1, 1)]
        diagnostic :warning, :invalid_escape_use, { :escape => escape }, range

        p = @ts - 1
        @root_lexer_state = @lexer.class.lex_en_expr_end;
                begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

       end
                end
        when 78 then
# line 946 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
        emit_character_constant

        @root_lexer_state = @lexer.class.lex_en_expr_end;       begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

       end
                end
        when 79 then
# line 964 "lib/parser/lexer-strings.rl"
                begin
 @te = p
p = p - 1; begin 
        p = @ts - 1
        @root_lexer_state = @lexer.class.lex_en_expr_end;
                begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

       end
                end
        when 80 then
# line 946 "lib/parser/lexer-strings.rl"
                begin
 begin p = (( @te))-1; end
 begin 
        emit_character_constant

        @root_lexer_state = @lexer.class.lex_en_expr_end;       begin
                p += 1
                _trigger_goto = true
                _goto_level = _out
                break
        end

       end
                end
        when 81 then
# line 972 "lib/parser/lexer-strings.rl"
                begin
 @te = p+1
 begin  raise 'bug'  end
                end
# line 4935 "lib/parser/lexer-strings.rb"
                        end # action switch
                end
        end
        if _trigger_goto
                next
        end
        end
        if _goto_level <= _again
        _acts = _lex_to_state_actions[ @cs]
        _nacts = _lex_actions[_acts]
        _acts += 1
        while _nacts > 0
                _nacts -= 1
                _acts += 1
                case _lex_actions[_acts - 1]
        when 23 then
# line 1 "NONE"
                begin
 @ts = nil;             end
# line 4955 "lib/parser/lexer-strings.rb"
                end # to state action switch
        end
        if _trigger_goto
                next
        end
        if  @cs == 0
                _goto_level = _out
                next
        end
        p += 1
        if p != pe
                _goto_level = _resume
                next
        end
        end
        if _goto_level <= _test_eof
        if p == eof
        if _lex_eof_trans[ @cs] > 0
                _trans = _lex_eof_trans[ @cs] - 1;
                _goto_level = _eof_trans
                next;
        end
        end
        end
        if _goto_level <= _out
                break
        end
        end
        end

# line 78 "lib/parser/lexer-strings.rl"
    # %

    # Ragel creates a local variable called `testEof` but it doesn't use
    # it in any assignment. This dead code is here to swallow the warning.
    # It has no runtime cost because Ruby doesn't produce any instructions from it.
    if false
      testEof
    end

    [p, @root_lexer_state]
  end
close_interp_on_current_literal(p) click to toggle source
# File lib/parser/lexer-strings.rb, line 5068
def close_interp_on_current_literal(p)
  current_literal = literal
  if current_literal
    if current_literal.end_interp_brace_and_try_closing
      if version?(18, 19)
        emit(:tRCURLY, '}'.freeze, p - 1, p)
        @lexer.cond.lexpop
        @lexer.cmdarg.lexpop
      else
        emit(:tSTRING_DEND, '}'.freeze, p - 1, p)
      end

      if current_literal.saved_herebody_s
        @herebody_s = current_literal.saved_herebody_s
      end

      continue_lexing(current_literal)

      return true
    end
  end
end
continue_lexing(current_literal) click to toggle source
# File lib/parser/lexer-strings.rb, line 5042
def continue_lexing(current_literal)
  @cs = next_state_for_literal(current_literal)
end
dedent_level() click to toggle source
# File lib/parser/lexer-strings.rb, line 5091
def dedent_level
  # We erase @dedent_level as a precaution to avoid accidentally
  # using a stale value.
  dedent_level, @dedent_level = @dedent_level, nil
  dedent_level
end
literal() click to toggle source
# File lib/parser/lexer-strings.rb, line 5046
def literal
  @literal_stack.last
end
next_state_for_literal(literal) click to toggle source
# File lib/parser/lexer-strings.rb, line 5014
def next_state_for_literal(literal)
  if literal.words? && literal.backslash_delimited?
    if literal.interpolate?
      self.class.lex_en_interp_backslash_delimited_words
    else
      self.class.lex_en_plain_backslash_delimited_words
    end
  elsif literal.words? && !literal.backslash_delimited?
    if literal.interpolate?
      self.class.lex_en_interp_words
    else
      self.class.lex_en_plain_words
    end
  elsif !literal.words? && literal.backslash_delimited?
    if literal.interpolate?
      self.class.lex_en_interp_backslash_delimited
    else
      self.class.lex_en_plain_backslash_delimited
    end
  else
    if literal.interpolate?
      self.class.lex_en_interp_string
    else
      self.class.lex_en_plain_string
    end
  end
end
on_newline(p) click to toggle source

This hook is triggered by “main” lexer on every newline character

# File lib/parser/lexer-strings.rb, line 5099
def on_newline(p)
  # After every heredoc was parsed, @herebody_s contains the
  # position of next token after all heredocs.
  if @herebody_s
    p = @herebody_s
    @herebody_s = nil
  end
  p
end
pop_literal() click to toggle source
# File lib/parser/lexer-strings.rb, line 5050
def pop_literal
  old_literal = @literal_stack.pop

  @dedent_level = old_literal.dedent_level

  if old_literal.type == :tREGEXP_BEG
    @root_lexer_state = @lexer.class.lex_en_inside_string

    # Fetch modifiers.
    self.class.lex_en_regexp_modifiers
  else
    @root_lexer_state = @lexer.class.lex_en_expr_end

    # Do nothing, yield to main lexer
    nil
  end
end
push_literal(*args) click to toggle source

LITERAL STACK ===

# File lib/parser/lexer-strings.rb, line 5008
def push_literal(*args)
  new_literal = Parser::Lexer::Literal.new(self, *args)
  @literal_stack.push(new_literal)
  @cs = next_state_for_literal(new_literal)
end
read_character_constant(p) click to toggle source
# File lib/parser/lexer-strings.rb, line 4998
def read_character_constant(p)
  @cs = self.class.lex_en_character

  advance(p)
end
reset() click to toggle source
# File lib/parser/lexer-strings.rb, line 3313
def reset
  @cs            = self.class.lex_en_unknown
  @literal_stack = []

  @escape_s      = nil # starting position of current sequence
  @escape        = nil # last escaped sequence, as string

  @herebody_s    = nil # starting position of current heredoc line

  # After encountering the closing line of <<~SQUIGGLY_HEREDOC,
  # we store the indentation level and give it out to the parser
  # on request. It is not possible to infer indentation level just
  # from the AST because escape sequences such as `\ ` or `\t` are
  # expanded inside the lexer, but count as non-whitespace for
  # indentation purposes.
  @dedent_level  = nil
end

Protected Instance Methods

check_ambiguous_slash(tm) click to toggle source
# File lib/parser/lexer-strings.rb, line 5396
def check_ambiguous_slash(tm)
  if tok(tm, tm + 1) == '/'.freeze
    # Ambiguous regexp literal.
    if @version < 30
      diagnostic :warning, :ambiguous_literal, nil, range(tm, tm + 1)
    else
      diagnostic :warning, :ambiguous_regexp, nil, range(tm, tm + 1)
    end
  end
end
check_invalid_escapes(p) click to toggle source
# File lib/parser/lexer-strings.rb, line 5407
def check_invalid_escapes(p)
  if emit_invalid_escapes?
    diagnostic :fatal, :invalid_unicode_escape, nil, range(@escape_s - 1, p)
  end
end
cond() click to toggle source
# File lib/parser/lexer-strings.rb, line 5135
def cond
  @lexer.cond
end
diagnostic(type, reason, arguments=nil, location=range, highlights=[]) click to toggle source
# File lib/parser/lexer-strings.rb, line 5131
def diagnostic(type, reason, arguments=nil, location=range, highlights=[])
  @lexer.send(:diagnostic, type, reason, arguments, location, highlights)
end
emit(type, value = tok, s = @ts, e = @te) click to toggle source
# File lib/parser/lexer-strings.rb, line 5127
def emit(type, value = tok, s = @ts, e = @te)
  @lexer.send(:emit, type, value, s, e)
end
emit_character_constant() click to toggle source
# File lib/parser/lexer-strings.rb, line 5386
def emit_character_constant
  value = @escape || tok(@ts + 1)

  if version?(18)
    emit(:tINTEGER, value.getbyte(0))
  else
    emit(:tCHARACTER, value)
  end
end
emit_interp_var(interp_var_kind) click to toggle source
# File lib/parser/lexer-strings.rb, line 5363
def emit_interp_var(interp_var_kind)
  case interp_var_kind
  when :cvar
    @lexer.send(:emit_class_var, @ts + 1, @te)
  when :ivar
    @lexer.send(:emit_instance_var, @ts + 1, @te)
  when :gvar
    @lexer.send(:emit_global_var, @ts + 1, @te)
  end
end
emit_invalid_escapes?() click to toggle source
# File lib/parser/lexer-strings.rb, line 5139
def emit_invalid_escapes?
  # always true for old Rubies
  return true if @version < 32

  # in "?\u123" case we don't push any literals
  # but we always emit invalid escapes
  return true if literal.nil?

  # Ruby >= 32, regexp, exceptional case
  !literal.regexp?
end
encode_escape(ord) click to toggle source
# File lib/parser/lexer-strings.rb, line 5290
def encode_escape(ord)
  ord.chr.force_encoding(source_buffer.source.encoding)
end
encode_escaped_char(p) click to toggle source
# File lib/parser/lexer-strings.rb, line 5374
def encode_escaped_char(p)
  @escape = encode_escape(tok(p - 2, p).to_i(16))
end
eof_codepoint?(point) click to toggle source
# File lib/parser/lexer-strings.rb, line 5111
def eof_codepoint?(point)
  [0x04, 0x1a, 0x00].include? point
end
extend_interp_code(current_literal) click to toggle source
# File lib/parser/lexer-strings.rb, line 5209
def extend_interp_code(current_literal)
  current_literal.flush_string
  current_literal.extend_content

  emit(:tSTRING_DBEG, '#{'.freeze)

  if current_literal.heredoc?
    current_literal.saved_herebody_s = @herebody_s
    @herebody_s = nil
  end

  current_literal.start_interp_brace
  @lexer.command_start = true
end
extend_interp_digit_var() click to toggle source
# File lib/parser/lexer-strings.rb, line 5224
def extend_interp_digit_var
  if @version >= 27
    literal.extend_string(tok, @ts, @te)
  else
    message = tok.start_with?('#@@') ? :cvar_name : :ivar_name
    diagnostic :error, message, { :name => tok(@ts + 1, @te) }, range(@ts + 1, @te)
  end
end
extend_interp_var(current_literal) click to toggle source
# File lib/parser/lexer-strings.rb, line 5354
def extend_interp_var(current_literal)
  current_literal.flush_string
  current_literal.extend_content

  emit(:tSTRING_DVAR, nil, @ts, @ts + 1)

  @ts
end
extend_string_eol_check_eof(current_literal, pe) click to toggle source
# File lib/parser/lexer-strings.rb, line 5233
def extend_string_eol_check_eof(current_literal, pe)
  if @te == pe
    diagnostic :fatal, :string_eof, nil,
               range(current_literal.str_s, current_literal.str_s + 1)
  end
end
extend_string_eol_heredoc_intertwined(p) click to toggle source
# File lib/parser/lexer-strings.rb, line 5250
def extend_string_eol_heredoc_intertwined(p)
  if @herebody_s
    # This is a regular literal intertwined with a heredoc. Like:
    #
    #     p <<-foo+"1
    #     bar
    #     foo
    #     2"
    #
    # which, incidentally, evaluates to "bar\n1\n2".
    p = @herebody_s - 1
    @herebody_s = nil
  end
  p
end
extend_string_eol_heredoc_line() click to toggle source
# File lib/parser/lexer-strings.rb, line 5240
def extend_string_eol_heredoc_line
  line = tok(@herebody_s, @ts).gsub(/\r+$/, ''.freeze)

  if version?(18, 19, 20)
    # See ruby:c48b4209c
    line = line.gsub(/\r.*$/, ''.freeze)
  end
  line
end
extend_string_eol_words(current_literal, p) click to toggle source
# File lib/parser/lexer-strings.rb, line 5266
def extend_string_eol_words(current_literal, p)
  if current_literal.words? && !eof_codepoint?(source_pts[p])
    current_literal.extend_space @ts, @te
  else
    # A literal newline is appended if the heredoc was _not_ closed
    # this time (see fbreak above). See also Literal#nest_and_try_closing
    # for rationale of calling #flush_string here.
    current_literal.extend_string tok, @ts, @te
    current_literal.flush_string
  end
end
extend_string_escaped() click to toggle source

String escaping

# File lib/parser/lexer-strings.rb, line 5153
def extend_string_escaped
  current_literal = literal
  # Get the first character after the backslash.
  escaped_char = source_buffer.slice(@escape_s, 1).chr

  if current_literal.munge_escape? escaped_char
    # If this particular literal uses this character as an opening
    # or closing delimiter, it is an escape sequence for that
    # particular character. Write it without the backslash.

    if current_literal.regexp? && REGEXP_META_CHARACTERS.match(escaped_char)
      # Regular expressions should include escaped delimiters in their
      # escaped form, except when the escaped character is
      # a closing delimiter but not a regexp metacharacter.
      #
      # The backslash itself cannot be used as a closing delimiter
      # at the same time as an escape symbol, but it is always munged,
      # so this branch also executes for the non-closing-delimiter case
      # for the backslash.
      current_literal.extend_string(tok, @ts, @te)
    else
      current_literal.extend_string(escaped_char, @ts, @te)
    end
  else
    # It does not. So this is an actual escape sequence, yay!
    if current_literal.squiggly_heredoc? && escaped_char == "\n".freeze
      # Squiggly heredocs like
      #   <<~-HERE
      #     1\
      #     2
      #   HERE
      # treat '\' as a line continuation, but still dedent the body, so the heredoc above becomes "12\n".
      # This information is emitted as is, without escaping,
      # later this escape sequence (\\\n) gets handled manually in the Lexer::Dedenter
      current_literal.extend_string(tok, @ts, @te)
    elsif current_literal.supports_line_continuation_via_slash? && escaped_char == "\n".freeze
      # Heredocs, regexp and a few other types of literals support line
      # continuation via \\\n sequence. The code like
      #   "a\
      #   b"
      # must be parsed as "ab"
      current_literal.extend_string(tok.gsub("\\\n".freeze, ''.freeze), @ts, @te)
    elsif current_literal.regexp? && @version >= 31 && %w[c C m M].include?(escaped_char)
      # Ruby >= 3.1 escapes \c- and \m chars, that's the only escape sequence
      # supported by regexes so far, so it needs a separate branch.
      current_literal.extend_string(@escape, @ts, @te)
    elsif current_literal.regexp?
      # Regular expressions should include escape sequences in their
      # escaped form. On the other hand, escaped newlines are removed (in cases like "\\C-\\\n\\M-x")
      current_literal.extend_string(tok.gsub("\\\n".freeze, ''.freeze), @ts, @te)
    else
      current_literal.extend_string(@escape || tok, @ts, @te)
    end
  end
end
extend_string_for_token_range(current_literal, string) click to toggle source
# File lib/parser/lexer-strings.rb, line 5286
def extend_string_for_token_range(current_literal, string)
  current_literal.extend_string(string, @ts, @te)
end
extend_string_slice_end(lookahead) click to toggle source
# File lib/parser/lexer-strings.rb, line 5278
def extend_string_slice_end(lookahead)
  # tLABEL_END is only possible in non-cond context on >= 2.2
  if @version >= 22 && !cond.active?
    lookahead = source_buffer.slice(@te, 2)
  end
  lookahead
end
range(s = @ts, e = @te) click to toggle source
# File lib/parser/lexer-strings.rb, line 5123
def range(s = @ts, e = @te)
  Parser::Source::Range.new(@source_buffer, s, e)
end
read_post_meta_or_ctrl_char(p) click to toggle source
# File lib/parser/lexer-strings.rb, line 5346
def read_post_meta_or_ctrl_char(p)
  @escape = source_buffer.slice(p - 1, 1).chr

  if @version >= 27 && ((0..8).include?(@escape.ord) || (14..31).include?(@escape.ord))
    diagnostic :fatal, :invalid_escape
  end
end
slash_c_char() click to toggle source
# File lib/parser/lexer-strings.rb, line 5378
def slash_c_char
  @escape = encode_escape(@escape[0].ord & 0x9f)
end
slash_m_char() click to toggle source
# File lib/parser/lexer-strings.rb, line 5382
def slash_m_char
  @escape = encode_escape(@escape[0].ord | 0x80)
end
tok(s = @ts, e = @te) click to toggle source
# File lib/parser/lexer-strings.rb, line 5119
def tok(s = @ts, e = @te)
  @source_buffer.slice(s, e - s)
end
unescape_char(p) click to toggle source
# File lib/parser/lexer-strings.rb, line 5294
def unescape_char(p)
  codepoint = source_pts[p - 1]

  if @version >= 30 && (codepoint == 117 || codepoint == 85) # 'u' or 'U'
    diagnostic :fatal, :invalid_escape
  end

  if (@escape = ESCAPES[codepoint]).nil?
    @escape = encode_escape(source_buffer.slice(p - 1, 1))
  end
end
unicode_points(p) click to toggle source
# File lib/parser/lexer-strings.rb, line 5306
def unicode_points(p)
  @escape = ""

  codepoints = tok(@escape_s + 2, p - 1)
  codepoint_s = @escape_s + 2

  if @version < 24
    if codepoints.start_with?(" ") || codepoints.start_with?("\t")
      diagnostic :fatal, :invalid_unicode_escape, nil,
                 range(@escape_s + 2, @escape_s + 3)
    end

    if spaces_p = codepoints.index(/[ \t]{2}/)
      diagnostic :fatal, :invalid_unicode_escape, nil,
                 range(codepoint_s + spaces_p + 1, codepoint_s + spaces_p + 2)
    end

    if codepoints.end_with?(" ") || codepoints.end_with?("\t")
      diagnostic :fatal, :invalid_unicode_escape, nil, range(p - 1, p)
    end
  end

  codepoints.scan(/([0-9a-fA-F]+)|([ \t]+)/).each do |(codepoint_str, spaces)|
    if spaces
      codepoint_s += spaces.length
    else
      codepoint = codepoint_str.to_i(16)

      if codepoint >= 0x110000
        diagnostic :error, :unicode_point_too_large, nil,
                   range(codepoint_s, codepoint_s + codepoint_str.length)
        break
      end

      @escape += codepoint.chr(Encoding::UTF_8)
      codepoint_s += codepoint_str.length
    end
  end
end
version?(*versions) click to toggle source
# File lib/parser/lexer-strings.rb, line 5115
def version?(*versions)
  versions.include?(@version)
end