class Soulheart::Loader
Public Class Methods
new(defaults={})
click to toggle source
# File lib/soulheart/loader.rb, line 3 def initialize(defaults={}) @no_all = defaults[:no_all] @no_combinatorial = defaults[:no_combinatorial] end
Public Instance Methods
add_item(item, category_base_id=nil, cleaned=false)
click to toggle source
# File lib/soulheart/loader.rb, line 116 def add_item(item, category_base_id=nil, cleaned=false) item = clean(item) unless cleaned category_base_id ||= category_id(item['category']) priority = (-item['priority']) redis.pipelined do redis.zadd(no_query_id(category_base_id), priority, item['term']) # Add to master set for queryless searches # store the raw data in a separate key to reduce memory usage, if it's cleaned it's done redis.hset(results_hashes_id, item['term'], item['data'].to_json) unless cleaned phrase = ([item['term']] + (item['aliases'] || [])).join(' ') # Store all the prefixes prefixes_for_phrase(phrase).each do |p| redis.sadd(Soulheart.base_id, p) unless cleaned # remember prefix in a master set # store the normalized term in the index for each of the categories redis.zadd("#{category_base_id}#{p}", priority, item['term']) end end item end
add_to_categories_array(category)
click to toggle source
# File lib/soulheart/loader.rb, line 22 def add_to_categories_array(category) if @no_combinatorial return if redis.smembers(hidden_categories_id).include?(category) redis.sadd hidden_categories_id, category elsif !redis.smembers(categories_id).include?(category) redis.sadd categories_id, category end end
clean(item)
click to toggle source
# File lib/soulheart/loader.rb, line 107 def clean(item) item = clean_hash(item) item.keys.select{ |k| !%w(category priority term aliases data).include?(k) }.each do |key| item['data'].merge!({"#{key}" => item.delete(key)}) end add_to_categories_array(item['category']) item end
clean_hash(item)
click to toggle source
# File lib/soulheart/loader.rb, line 98 def clean_hash(item) item['aliases'] = item['aliases'].split(',').map(&:strip) if item['aliases'] && !item['aliases'].kind_of?(Array) fail ArgumentError, 'Items must have text' unless item['text'] default_items_hash(item.delete('text'), item.delete('category')) .tap { |i| i['data'].merge!(item.delete('data')) if item['data'] } .tap { |i| i['priority'] = item.delete('priority').to_f if item['priority'] } .merge item end
clear(should_clear_cache = false)
click to toggle source
# File lib/soulheart/loader.rb, line 66 def clear(should_clear_cache = false) clear_cache if should_clear_cache category_combos.each {|cat| delete_data(category_id(cat)) } delete_categories delete_data end
clear_cache()
click to toggle source
# File lib/soulheart/loader.rb, line 58 def clear_cache # Remove the remove_results_hash # has to be called before the cat_ids are cleared category_combos.map { |cat| redis.expire(no_query_id(category_id(cat)), 0) } redis.expire results_hashes_id, 0 redis.del(results_hashes_id) end
default_items_hash(text, category)
click to toggle source
# File lib/soulheart/loader.rb, line 8 def default_items_hash(text, category) category ||= 'default' { 'category' => normalize(category), 'priority' => 100, 'term' => normalize(text), 'aliases' => [], 'data' => { 'text' => text, 'category' => category } } end
delete_categories()
click to toggle source
# File lib/soulheart/loader.rb, line 31 def delete_categories redis.expire category_combos_id, 0 redis.expire categories_id, 0 redis.expire hidden_categories_id, 0 end
delete_data(id="
click to toggle source
# File lib/soulheart/loader.rb, line 42 def delete_data(id="#{Soulheart.base_id}:") # delete the sorted sets for this type phrases = redis.smembers(Soulheart.base_id) redis.pipelined do phrases.each do |p| redis.del("#{id}#{p}") end redis.del(id) end # Redis can continue serving cached requests while the reload is # occurring. Some requests may be cached incorrectly as empty set (for requests # which come in after the above delete, but before the loading completes). But # everything will work itself out as soon as the cache expires again. end
load(items)
click to toggle source
# File lib/soulheart/loader.rb, line 73 def load(items) Soulheart.stop_words # Load stop words so we don't pipeline redis_stop_words accidentally i = 0 items.each do |item| item.replace(add_item(item)) # Replace with item return so we know we have category_id i += 1 end set_category_combos_array.each do |category_combo| items.each do |item| if category_combo == item['category'] next elsif category_combo == 'all' next if @no_all elsif @no_combinatorial next elsif !category_combo.match(item['category']) next end add_item(item, category_id(category_combo), true) # send it base i += 1 end end puts "Total items (including combinatorial categories): #{i}" end
reset_categories(categories)
click to toggle source
# File lib/soulheart/loader.rb, line 37 def reset_categories(categories) delete_categories redis.sadd categories_id, categories end