module WordTokenizer

Public Instance Methods

tokenize(s) click to toggle source
# File lib/word_tokenizer.rb, line 50
def tokenize(s)
  rules = []
  @@tokenize_regexps.each {|rules| s.gsub!(rules[0], rules[1])}
end