class Tagfish::Tokeniser

Public Class Methods

dump(tokens) click to toggle source
# File lib/tagfish/tokeniser.rb, line 33
def self.dump(tokens)
  tokens.join('')
end
tokenise(rest) click to toggle source
# File lib/tagfish/tokeniser.rb, line 16
def self.tokenise(rest)
  tokens = []
  while true
    match = rest.match /[\w\/:.-]+\/[\w.-]+:[\w.-]+/
    if match.nil?
      tokens << Text.new(rest)
      break
    else
      tokens << Text.new(match.pre_match)
      tokens << URI.new(match.to_s)
      rest = match.post_match
    end
  end

  tokens
end