class Mitier::Tokenizer
Attributes
text[RW]
tokens[RW]
Public Class Methods
new(text)
click to toggle source
# File lib/mitier/tokenizer.rb, line 5 def initialize(text) @text = text.to_s.strip end
Public Instance Methods
process()
click to toggle source
# File lib/mitier/tokenizer.rb, line 9 def process return [] if text.empty? tokens_ptr = tokenize process_token_elements tokens_ptr end
Private Instance Methods
process_token_elements(ptr)
click to toggle source
# File lib/mitier/tokenizer.rb, line 21 def process_token_elements(ptr) @tokens = [].tap do |elements| until (element = ptr.read_pointer).null? elements << element.read_string ptr += FFI::Type::POINTER.size end end end
tokenize()
click to toggle source
# File lib/mitier/tokenizer.rb, line 17 def tokenize Wrapper.mitie_tokenize text end