class Mitier::Tokenizer

Attributes

text[RW]
tokens[RW]

Public Class Methods

new(text) click to toggle source
# File lib/mitier/tokenizer.rb, line 5
def initialize(text)
  @text = text.to_s.strip
end

Public Instance Methods

process() click to toggle source
# File lib/mitier/tokenizer.rb, line 9
def process
  return [] if text.empty?
  tokens_ptr = tokenize
  process_token_elements tokens_ptr
end

Private Instance Methods

process_token_elements(ptr) click to toggle source
# File lib/mitier/tokenizer.rb, line 21
def process_token_elements(ptr)
  @tokens = [].tap do |elements|
    until (element = ptr.read_pointer).null?
      elements << element.read_string
      ptr += FFI::Type::POINTER.size
    end
  end
end
tokenize() click to toggle source
# File lib/mitier/tokenizer.rb, line 17
def tokenize
  Wrapper.mitie_tokenize text
end