module Mongoid::FTS::Util
Public Instance Methods
boolean_and(*strings)
click to toggle source
# File lib/mongoid-fts/util.rb, line 307 def boolean_and(*strings) strings = Coerce.list_of_strings(*strings) strings.map{|s| '"%s"' % s.gsub('"', '')}.join(' ') end
boolean_or(*strings)
click to toggle source
# File lib/mongoid-fts/util.rb, line 312 def boolean_or(*strings) strings = Coerce.list_of_strings(*strings) strings.join(' ') end
chars(string)
click to toggle source
# File lib/mongoid-fts/util.rb, line 209 def chars(string) chars = [] UnicodeUtils.each_grapheme(string.to_s){|g| chars.push(g)} chars end
connect!()
click to toggle source
# File lib/mongoid-fts/util.rb, line 301 def connect! Mongoid.configure do |config| config.connect_to('mongoid-fts') end end
create_indexes()
click to toggle source
# File lib/mongoid-fts/util.rb, line 34 def create_indexes fts_models.each{|model| model.create_indexes} end
destroy_all()
click to toggle source
# File lib/mongoid-fts/util.rb, line 38 def destroy_all fts_models.map{|model| model.destroy_all} end
enable!(*args)
click to toggle source
# File lib/mongoid-fts/util.rb, line 279 def enable!(*args) options = Map.options_for!(args) unless options.has_key?(:warn) options[:warn] = true end begin session = Mongoid::Sessions.default session.with(database: :admin).command({ setParameter: 1, textSearchEnabled: true }) rescue Object => e unless e.is_a?(Mongoid::Errors::NoSessionsConfig) warn "failed to enable search with #{ e.class }(#{ e.message })" end end end
find_in_batches(queries = {})
click to toggle source
# File lib/mongoid-fts/util.rb, line 43 def find_in_batches(queries = {}) models = queries.map do |model_class, model_ids| unless model_class.is_a?(Class) model_class = eval(model_class.to_s) end model_ids = Array(model_ids) begin model_class.find(model_ids) rescue Mongoid::Errors::DocumentNotFound model_ids.map do |model_id| begin model_class.find(model_id) rescue Mongoid::Errors::DocumentNotFound nil end end end end models.flatten! models.compact! models end
find_or_create(finder, creator)
click to toggle source
# File lib/mongoid-fts/util.rb, line 70 def find_or_create(finder, creator) doc = finder.call() return doc if doc n, max = 0, 2 begin creator.call() rescue Object => e n += 1 raise if n > max sleep(rand(0.1)) finder.call() or retry end end
fts_models()
click to toggle source
# File lib/mongoid-fts/util.rb, line 5 def fts_models [ Mongoid::FTS::Index ] end
fuzzy(*args)
click to toggle source
# File lib/mongoid-fts/util.rb, line 166 def fuzzy(*args) strings = Coerce.list_of_strings(args).map{|string| utf8ify(string)} list = [] strings.each do |string| list.push(*ngrams_for(string)) decoded = unidecode(string) unless decoded == string list.push(*ngrams_for(decoded)) end end list.uniq end
Also aliased as: fuzzy_for
index(*args, &block)
click to toggle source
# File lib/mongoid-fts/util.rb, line 238 def index(*args, &block) if args.empty? and block.nil? Index else args.each do |arg| case arg when Class arg.all.each{|model| Index.add(model)} else Index.add(arg, &block) end end end end
index!(*args, &block)
click to toggle source
# File lib/mongoid-fts/util.rb, line 257 def index!(*args, &block) Index.add!(*args, &block) end
list_of_strings(*args)
click to toggle source
# File lib/mongoid-fts/util.rb, line 233 def list_of_strings(*args) args.flatten.compact.map{|arg| arg.to_s}.select{|arg| !arg.empty?}.uniq end
literals_for(*args)
click to toggle source
# File lib/mongoid-fts/util.rb, line 148 def literals_for(*args) words = FTS.normalized_array(args) return words.map{|word| "__#{ Digest::MD5.hexdigest(word) }__"} end
models()
click to toggle source
# File lib/mongoid-fts/util.rb, line 266 def models @models ||= [] end
ngrams_for(*args)
click to toggle source
# File lib/mongoid-fts/util.rb, line 185 def ngrams_for(*args) options = Map.options_for!(args) strings = Coerce.list_of_strings(args).map{|string| utf8ify(string)} list = [] sizes = options[:sizes] || [2,3] strings.each do |string| chars = Util.chars('_' + string + '_') sizes.each do |size| (chars.size - (size - 1)).times do |i| ngram = chars[i, size].join list.push(ngram) end end end list end
normalized_array(*array)
click to toggle source
# File lib/mongoid-fts/util.rb, line 229 def normalized_array(*array) array.flatten.map{|_| _.to_s.strip}.select{|_| !_.empty?}.uniq end
reset!()
click to toggle source
# File lib/mongoid-fts/util.rb, line 11 def reset! Mongoid::FTS.setup!(:warn => true) fts_models.each do |model| model.destroy_all begin model.collection.indexes.drop rescue Object => e end begin model.collection.drop rescue Object => e end begin model.create_indexes rescue Object => e end end end
session()
click to toggle source
# File lib/mongoid-fts/util.rb, line 271 def session @session ||= Mongoid::Sessions.default end
session=(session)
click to toggle source
# File lib/mongoid-fts/util.rb, line 275 def session=(session) @session = session end
setup!(*args)
click to toggle source
# File lib/mongoid-fts/util.rb, line 296 def setup!(*args) enable!(*args) Index.setup! end
stems_for(*args, &block)
click to toggle source
# File lib/mongoid-fts/util.rb, line 140 def stems_for(*args, &block) options = Map.options_for!(args) words = Coerce.list_of_strings(*args).map{|word| utf8ify(word)} Stemming.stem(*words) end
stopword?(word)
click to toggle source
# File lib/mongoid-fts/util.rb, line 154 def stopword?(word) word = utf8ify(word) word.empty? or Stemming::Stopwords.stopword?(word) end
strip(word)
click to toggle source
# File lib/mongoid-fts/util.rb, line 159 def strip(word) word = utf8ify(word) word.gsub!(/\A(?:[^\w]|_|\s)+/, '') # leading punctuation/spaces word.gsub!(/(?:[^\w]|_|\s+)+\Z/, '') # trailing punctuation/spaces word end
terms_for(*args, &block)
click to toggle source
# File lib/mongoid-fts/util.rb, line 87 def terms_for(*args, &block) options = Map.options_for!(args) words = words_for(*args) list = options[:list] || [] words.each do |word| word = word.downcase next if stopword?(word) stems = stems_for(word) stems.each do |stem| [stem, unidecode(stem)].uniq.each do |stem| next if stopword?(stem) block ? block.call(stem) : list.push(stem) substems = stem.split(/_/) if options[:subterms] and substems.size > 1 substems.each do |substem| terms_for(substem.gsub(/_+/, '-'), :list => list) end end end end end list.uniq! block ? nil : list end
unidecode(string)
click to toggle source
# File lib/mongoid-fts/util.rb, line 215 def unidecode(string) Stringex::Unidecoder.decode(utf8ify(string.to_s)) end
unindex(*args, &block)
click to toggle source
# File lib/mongoid-fts/util.rb, line 253 def unindex(*args, &block) Index.remove(*args, &block) end
unindex!(*args, &block)
click to toggle source
# File lib/mongoid-fts/util.rb, line 261 def unindex!(*args, &block) Index.remove!(*args, &block) end
utf8ify(string)
click to toggle source
# File lib/mongoid-fts/util.rb, line 219 def utf8ify(string) UnicodeUtils.nfkd( begin string.force_encoding('UTF-8') rescue string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') end ) end
words_for(*args, &block)
click to toggle source
# File lib/mongoid-fts/util.rb, line 122 def words_for(*args, &block) options = Map.options_for!(args) string = args.join(' ') list = [] UnicodeUtils.each_word(string) do |word| word = strip(utf8ify(word)) next if word.empty? block ? block.call(word) : list.push(word) end block ? nil : list end