class WirisPlugin::Tokenizer

Attributes

delimiters[RW]
takeDelimiters[RW]

Public Class Methods

new(delimiters, takeDelimiters) click to toggle source
Calls superclass method
# File lib/com/wiris/util/type/Tokenizer.rb, line 8
def initialize(delimiters, takeDelimiters)
    super()
    self.delimiters = delimiters
    self.takeDelimiters = takeDelimiters
end

Public Instance Methods

getDelimiters() click to toggle source
# File lib/com/wiris/util/type/Tokenizer.rb, line 13
def getDelimiters()
    return self.delimiters
end
splitTokens(text) click to toggle source
# File lib/com/wiris/util/type/Tokenizer.rb, line 16
def splitTokens(text)
    tokens = Array.new()
    it = Utf8::getIterator(text)
    block = StringBuf.new()
    while it::hasNext()
        c = Utf8::uchr(it::next())
        if self.delimiters::indexOf(c) >= 0
            tokens::push(block::toString())
            if self.takeDelimiters
                tokens::push(c)
            end
            block = StringBuf.new()
        else 
            block::add(c)
        end
    end
    last = block::toString()
    if !(last == "")
        tokens::push(last)
    end
    return tokens
end