class WirisPlugin::Tokenizer
Attributes
delimiters[RW]
takeDelimiters[RW]
Public Class Methods
new(delimiters, takeDelimiters)
click to toggle source
Calls superclass method
# File lib/com/wiris/util/type/Tokenizer.rb, line 8 def initialize(delimiters, takeDelimiters) super() self.delimiters = delimiters self.takeDelimiters = takeDelimiters end
Public Instance Methods
getDelimiters()
click to toggle source
# File lib/com/wiris/util/type/Tokenizer.rb, line 13 def getDelimiters() return self.delimiters end
splitTokens(text)
click to toggle source
# File lib/com/wiris/util/type/Tokenizer.rb, line 16 def splitTokens(text) tokens = Array.new() it = Utf8::getIterator(text) block = StringBuf.new() while it::hasNext() c = Utf8::uchr(it::next()) if self.delimiters::indexOf(c) >= 0 tokens::push(block::toString()) if self.takeDelimiters tokens::push(c) end block = StringBuf.new() else block::add(c) end end last = block::toString() if !(last == "") tokens::push(last) end return tokens end