class Jrr::Tokenizer

Constants

LPAREN
RPAREN

Attributes

case_sensitive[R]

Public Instance Methods

last_token() click to toggle source
# File lib/jrr/tokenizer.rb, line 35
def last_token
  @tokens.last
end
scan(string, scanner) click to toggle source
# File lib/jrr/tokenizer.rb, line 39
def scan(string, scanner)
  if tokens = scanner.scan(string, last_token)
    tokens.each do |token|
      if token.empty?
        fail! :unexpected_zero_width_match,
              token_category: token.category, at: string
      end

      @nesting += 1 if LPAREN == token
      @nesting -= 1 if RPAREN == token
      fail! :too_many_closing_parentheses if @nesting < 0

      @tokens << token unless token.is?(:whitespace)
    end

    match_length = tokens.map(&:length).reduce(:+)
    [true, string[match_length..-1]]
  else
    [false, string]
  end
end
strip_comments(input) click to toggle source
# File lib/jrr/tokenizer.rb, line 61
def strip_comments(input)
  input.gsub(/\/\*[^*]*\*+(?:[^*\/][^*]*\*+)*\//, '')
end
tokenize(string, options = {}) click to toggle source
# File lib/jrr/tokenizer.rb, line 13
def tokenize(string, options = {})
  @nesting = 0
  @tokens  = []
  input    = strip_comments(string.to_s.dup)
  @case_sensitive = options.fetch(:case_sensitive, false)

  until input.empty?
    scanned = Scanner.scanners(case_sensitive: case_sensitive).any? do |scanner|
      scanned, input = scan(input, scanner)
      scanned
    end

    unless scanned
      fail! :parse_error, at: input
    end
  end

  fail! :too_many_opening_parentheses if @nesting > 0

  @tokens
end

Private Instance Methods

fail!(reason, **meta) click to toggle source
# File lib/jrr/tokenizer.rb, line 67
def fail!(reason, **meta)
  message =
    case reason
    when :parse_error
      "parse error at: '#{meta.fetch(:at)}'"
    when :too_many_opening_parentheses
      "too many opening parentheses"
    when :too_many_closing_parentheses
      "too many closing parentheses"
    when :unexpected_zero_width_match
      "unexpected zero-width match (:#{meta.fetch(:category)}) at '#{meta.fetch(:at)}'"
    else
      raise ::ArgumentError, "Unhandled #{reason}"
    end

  raise TokenizerError.for(reason, meta), message
end