class DaimonSkycrawlers::Consumer::URL

URL consumer class

Public Class Methods

crawlers() click to toggle source

Returns registered crawlers

@return [Array<Crawler>]

# File lib/daimon_skycrawlers/consumer/url.rb, line 29
def crawlers
  @crawlers ||= []
end
queue_name() click to toggle source

@private

# File lib/daimon_skycrawlers/consumer/url.rb, line 36
def queue_name
  "#{DaimonSkycrawlers.configuration.queue_name_prefix}.url"
end
register(crawler) click to toggle source

Register a given crawler

@param crawler [Crawler] crawler instance which implements `fetch` method @return [void]

# File lib/daimon_skycrawlers/consumer/url.rb, line 20
def register(crawler)
  crawlers << crawler
end

Public Instance Methods

process(message) click to toggle source

@private

# File lib/daimon_skycrawlers/consumer/url.rb, line 46
def process(message)
  crawler_interval = DaimonSkycrawlers.configuration.crawler_interval

  # XXX When several crawlers are registered, how should they behave?
  self.class.crawlers.each do |crawler|
    crawler.process(message.dup)
    if crawler.skipped?
      sleep(crawler_interval) if crawler.n_processed_urls % 50 == 0
    else
      sleep(crawler_interval)
    end
  end
end