class Shuwar::Tokenizer

The tokenizer

Public Class Methods

new(f) click to toggle source

Make a new tokenizer that reads input from f

f should respond to each_line

# File lib/shuwar/tokenizer.rb, line 12
def initialize(f)
  @input = f
  @stack = []
end

Public Instance Methods

alter_stack(to, &block) click to toggle source

Change stack to the specified contents by yielding an open paren and a symbol when push is needed and yielding a close paren when pop is needed.

Make sure you call +alter_stack [], block+ before you go

# File lib/shuwar/tokenizer.rb, line 79
def alter_stack(to, &block)
  stack_tail, to_tail = @stack.dup, to.dup

  begin
    while stack_tail.fetch(0) == to_tail.fetch(0)
      stack_tail.shift
      to_tail.shift
    end
  rescue
    # No more items in one tail. End it.
  end

  stack_tail.size.times do
    block.call CloseParen.new
    @stack.pop
  end

  to_tail.each do |t|
    @stack.push t
    block.call OpenParen.new
    block.call t.to_sym
  end
end
each_input_line(&block) click to toggle source

Just a helper to chomp the line before +each_line+ing

# File lib/shuwar/tokenizer.rb, line 19
def each_input_line(&block)
  @input.each_line {|l| block.call l.chomp }
end
each_token(&block) click to toggle source

Iterates through each line and yield tokens. You should call this

# File lib/shuwar/tokenizer.rb, line 25
def each_token(&block)
  each_input_line do |line|
    each_token_in_line line, &block
  end

  alter_stack [], &block
end
each_token_in_line(line, &block) click to toggle source

Tokenize one line

# File lib/shuwar/tokenizer.rb, line 35
def each_token_in_line(line, &block)
  sp = line.split "|", 2
  if sp.size != 2
    alter_stack [], &block
    for c in %w{[ ] #}
      line.replace line.gsub c, " #{c} "
    end
    tags = line.split
    tags.each &:strip!
    tags.reject! &:empty?
    tags.each do |t|
      case
        when t == "["
          block.call OpenParen.new
        when t == "]"
          block.call CloseParen.new
        when t == "#"
          block.call Quote.new
        when t == "nil"
          block.call nil
        when t.chars.all? {|c| ('1'..'9') === c }
          block.call t.to_i
        when t.chars.all? {|c| ('1'..'9') === c or c == '.' }
          block.call t.to_f
        else
          block.call t.to_sym
      end
    end
  else
    ts, txt = sp
    tags = ts.split ":"
    tags.each &:strip!
    tags.reject! &:empty?
    alter_stack tags, &block
    block.call txt.strip
  end
end