class Oga::CSS::Lexer

Lexer for turning CSS expressions into a sequence of tokens. Tokens are returned as arrays with every array having two values:

  1. The token type as a Symbol

  2. The token value, or nil if there is no value.

## Thread Safety

Similar to the XPath lexer this lexer keeps track of an internal state. As a result it’s not safe to share the same instance of this lexer between multiple threads. However, no global state is used so you can use separate instances in threads just fine.

@api private

Attributes

_css_lexer_eof_trans[RW]
_css_lexer_from_state_actions[RW]
_css_lexer_index_offsets[RW]
_css_lexer_indicies[RW]
_css_lexer_key_spans[RW]
_css_lexer_to_state_actions[RW]
_css_lexer_trans_actions[RW]
_css_lexer_trans_keys[RW]
_css_lexer_trans_targs[RW]
css_lexer_en_main[RW]
css_lexer_en_predicate[RW]
css_lexer_en_pseudo_args[RW]
css_lexer_error[RW]
css_lexer_first_final[RW]
css_lexer_start[RW]

Public Class Methods

new(data) click to toggle source

@param [String] data The data to lex.

# File lib/oga/css/lexer.rb, line 376
def initialize(data)
  @data = data.strip
end

Public Instance Methods

advance(&block) click to toggle source

Advances through the input and generates the corresponding tokens. Each token is yielded to the supplied block.

This method stores the supplied block in ‘@block` and resets it after the lexer loop has finished.

@see [#add_token]

# File lib/oga/css/lexer.rb, line 401
      def advance(&block)
        @block   = block
        @escaped = false

        data  = @data # saves ivar lookups while lexing.
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.css_lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _css_lexer_eof_trans          = self.class.send(:_css_lexer_eof_trans)
        _css_lexer_from_state_actions = self.class.send(:_css_lexer_from_state_actions)
        _css_lexer_index_offsets      = self.class.send(:_css_lexer_index_offsets)
        _css_lexer_indicies           = self.class.send(:_css_lexer_indicies)
        _css_lexer_key_spans          = self.class.send(:_css_lexer_key_spans)
        _css_lexer_to_state_actions   = self.class.send(:_css_lexer_to_state_actions)
        _css_lexer_trans_actions      = self.class.send(:_css_lexer_trans_actions)
        _css_lexer_trans_keys         = self.class.send(:_css_lexer_trans_keys)
        _css_lexer_trans_targs        = self.class.send(:_css_lexer_trans_targs)

        
# line 428 "lib/oga/css/lexer.rb"
begin
        testEof = false
        _slen, _trans, _keys, _inds, _acts, _nacts = nil
        _goto_level = 0
        _resume = 10
        _eof_trans = 15
        _again = 20
        _test_eof = 30
        _out = 40
        while true
        if _goto_level <= 0
        if p == pe
                _goto_level = _test_eof
                next
        end
        if cs == 0
                _goto_level = _out
                next
        end
        end
        if _goto_level <= _resume
        case _css_lexer_from_state_actions[cs] 
        when 14 then
# line 1 "NONE"
                begin
ts = p
                end
# line 456 "lib/oga/css/lexer.rb"
        end
        _keys = cs << 1
        _inds = _css_lexer_index_offsets[cs]
        _slen = _css_lexer_key_spans[cs]
        _wide = ( (data.getbyte(p) || 0))
        _trans = if (   _slen > 0 && 
                        _css_lexer_trans_keys[_keys] <= _wide && 
                        _wide <= _css_lexer_trans_keys[_keys + 1] 
                    ) then
                        _css_lexer_indicies[ _inds + _wide - _css_lexer_trans_keys[_keys] ] 
                 else 
                        _css_lexer_indicies[ _inds + _slen ]
                 end
        end
        if _goto_level <= _eof_trans
        cs = _css_lexer_trans_targs[_trans]
        if _css_lexer_trans_actions[_trans] != 0
        case _css_lexer_trans_actions[_trans]
        when 23 then
# line 151 "lib/oga/css/lexer.rl"
                begin
 @escaped = true                end
        when 2 then
# line 1 "NONE"
                begin
te = p+1
                end
        when 36 then
# line 253 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin  add_token(:T_NTH)  end
                end
        when 6 then
# line 255 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin  add_token(:T_ODD)  end
                end
        when 5 then
# line 256 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin  add_token(:T_EVEN)  end
                end
        when 33 then
# line 235 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin 
          add_token(:T_RPAREN)

          cs = 16;
         end
                end
        when 34 then
# line 158 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 39 then
# line 246 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1;              end
        when 42 then
# line 254 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin  add_token(:T_MINUS)  end
                end
        when 41 then
# line 196 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin 
          value = slice_input(ts, te).to_i

          add_token(:T_INT, value)
         end
                end
        when 37 then
# line 158 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 3 then
# line 158 "lib/oga/css/lexer.rl"
                begin
 begin p = ((te))-1; end
 begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 4 then
# line 1 "NONE"
                begin
        case act
        when 0 then
        begin  begin
                cs = 0
                _goto_level = _again
                next
        end
end
        when 4 then
        begin begin p = ((te))-1; end
 add_token(:T_MINUS) end
end 
                        end
        when 45 then
# line 294 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin  add_token(:T_EQ)  end
                end
        when 12 then
# line 295 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin  add_token(:T_SPACE_IN)  end
                end
        when 10 then
# line 296 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin  add_token(:T_STARTS_WITH)  end
                end
        when 9 then
# line 297 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin  add_token(:T_ENDS_WITH)  end
                end
        when 50 then
# line 298 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin  add_token(:T_IN)  end
                end
        when 11 then
# line 299 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin  add_token(:T_HYPHEN_IN)  end
                end
        when 46 then
# line 276 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin 
          add_token(:T_RBRACK)

          cs = 16;
         end
                end
        when 8 then
# line 214 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin 
          emit(:T_STRING, ts + 1, te - 1)
         end
                end
        when 49 then
# line 284 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1;              end
        when 47 then
# line 158 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 7 then
# line 158 "lib/oga/css/lexer.rl"
                begin
 begin p = ((te))-1; end
 begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 18 then
# line 270 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin 
          add_token(:T_LBRACK)

          cs = 36;
         end
                end
        when 19 then
# line 134 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin 
          add_token(:T_PIPE)
         end
                end
        when 16 then
# line 229 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin 
          add_token(:T_LPAREN)

          cs = 27;
         end
                end
        when 17 then
# line 158 "lib/oga/css/lexer.rl"
                begin
te = p+1
 begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 15 then
# line 319 "lib/oga/css/lexer.rl"
                begin
te = p+1
                end
        when 27 then
# line 308 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin  add_token(:T_GREATER)  end
                end
        when 25 then
# line 309 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin  add_token(:T_PLUS)  end
                end
        when 28 then
# line 310 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin  add_token(:T_TILDE)  end
                end
        when 26 then
# line 138 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin 
          add_token(:T_COMMA)
         end
                end
        when 24 then
# line 124 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin 
          add_token(:T_SPACE)
         end
                end
        when 20 then
# line 158 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 1 then
# line 158 "lib/oga/css/lexer.rl"
                begin
 begin p = ((te))-1; end
 begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 40 then
# line 128 "lib/oga/css/lexer.rl"
                begin
 add_token(:T_HASH)             end
# line 248 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1;              end
        when 29 then
# line 128 "lib/oga/css/lexer.rl"
                begin
 add_token(:T_HASH)             end
# line 306 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1;              end
        when 43 then
# line 129 "lib/oga/css/lexer.rl"
                begin
 add_token(:T_DOT)              end
# line 248 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1;              end
        when 30 then
# line 129 "lib/oga/css/lexer.rl"
                begin
 add_token(:T_DOT)              end
# line 306 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1;              end
        when 44 then
# line 130 "lib/oga/css/lexer.rl"
                begin
 add_token(:T_COLON)            end
# line 248 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1;              end
        when 31 then
# line 130 "lib/oga/css/lexer.rl"
                begin
 add_token(:T_COLON)            end
# line 306 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1;              end
        when 38 then
# line 151 "lib/oga/css/lexer.rl"
                begin
 @escaped = true                end
# line 158 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 48 then
# line 151 "lib/oga/css/lexer.rl"
                begin
 @escaped = true                end
# line 158 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 21 then
# line 151 "lib/oga/css/lexer.rl"
                begin
 @escaped = true                end
# line 158 "lib/oga/css/lexer.rl"
                begin
te = p
p = p - 1; begin 
          value = slice_input(ts, te)

          # Translates "foo\.bar" into "foo.bar"
          if @escaped
            value    = value.gsub('\.', '.')
            @escaped = false
          end

          add_token(:T_IDENT, value)
         end
                end
        when 22 then
# line 1 "NONE"
                begin
te = p+1
                end
# line 151 "lib/oga/css/lexer.rl"
                begin
 @escaped = true                end
        when 35 then
# line 1 "NONE"
                begin
te = p+1
                end
# line 254 "lib/oga/css/lexer.rl"
                begin
act = 4;                end
# line 924 "lib/oga/css/lexer.rb"
        end
        end
        end
        if _goto_level <= _again
        case _css_lexer_to_state_actions[cs] 
        when 13 then
# line 1 "NONE"
                begin
ts = nil;               end
        when 32 then
# line 1 "NONE"
                begin
ts = nil;               end
# line 1 "NONE"
                begin
act = 0
                end
# line 942 "lib/oga/css/lexer.rb"
        end

        if cs == 0
                _goto_level = _out
                next
        end
        p += 1
        if p != pe
                _goto_level = _resume
                next
        end
        end
        if _goto_level <= _test_eof
        if p == eof
        if _css_lexer_eof_trans[cs] > 0
                _trans = _css_lexer_eof_trans[cs] - 1;
                _goto_level = _eof_trans
                next;
        end
        end

        end
        if _goto_level <= _out
                break
        end
end
        end

# line 76 "lib/oga/css/lexer.rl"

        # % fix highlight
      ensure
        @block = nil
      end
lex() click to toggle source

Gathers all the tokens for the input and returns them as an Array.

@see [#advance] @return [Array]

# File lib/oga/css/lexer.rb, line 384
def lex
  tokens = []

  advance do |type, value|
    tokens << [type, value]
  end

  return tokens
end

Private Instance Methods

add_token(type, value = nil) click to toggle source

Yields a new token to the supplied block.

@param [Symbol] type The token type. @param [String] value The token value.

@yieldparam [Symbol] type @yieldparam [String|NilClass] value

# File lib/oga/css/lexer.rb, line 1010
def add_token(type, value = nil)
  @block.call(type, value)
end
emit(type, start, stop) click to toggle source

Emits a token of which the value is based on the supplied start/stop position.

@param [Symbol] type The token type. @param [Fixnum] start @param [Fixnum] stop

@see [#text] @see [#add_token]

# File lib/oga/css/lexer.rb, line 988
def emit(type, start, stop)
  value = slice_input(start, stop)

  add_token(type, value)
end
slice_input(start, stop) click to toggle source

Returns the text between the specified start and stop position.

@param [Fixnum] start @param [Fixnum] stop @return [String]

# File lib/oga/css/lexer.rb, line 999
def slice_input(start, stop)
  return @data.byteslice(start, stop - start)
end