class Polipus::QueueOverflow::Manager
Attributes
polipus[R]
url_filter[RW]
Public Class Methods
new(polipus, main_q, item_limit)
click to toggle source
# File lib/polipus/queue_overflow/manager.rb, line 7 def initialize(polipus, main_q, item_limit) @polipus = polipus @main_q = main_q @adapter = @polipus.queue_overflow_adapter @item_limit = item_limit @redis = @polipus.redis end
Public Instance Methods
perform()
click to toggle source
# File lib/polipus/queue_overflow/manager.rb, line 19 def perform removed = 0 restored = 0 main_q_size = @main_q.size if main_q_size > @item_limit @polipus.logger.info { "Overflow Manager: Going to offload items from redis: ~#{main_q_size - @item_limit}" } removed = rotate(@main_q, @adapter) { @main_q.size > @item_limit } elsif main_q_size < @item_limit && !@adapter.empty? @polipus.logger.info { "Overflow Manager: Going to restore items into redis: ~#{@item_limit - main_q_size}" } restored = rotate(@adapter, @main_q) { @main_q.size <= @item_limit } end [removed, restored] end
Private Instance Methods
rotate(source, dest) { |source, dest| ... }
click to toggle source
# File lib/polipus/queue_overflow/manager.rb, line 35 def rotate(source, dest) performed = 0 loop do message = source.pop(true) if message page = Page.from_json message unless @polipus.storage.exists?(page) allowed = @url_filter.nil? ? true : @url_filter.call(page) if allowed dest << message performed += 1 end end end source.commit if source.respond_to? :commit break if !message || source.empty? break unless yield source, dest end performed end