module WordTokenizer
Public Instance Methods
tokenize(s)
click to toggle source
# File lib/word_tokenizer.rb, line 50 def tokenize(s) rules = [] @@tokenize_regexps.each {|rules| s.gsub!(rules[0], rules[1])} end