class DaimonSkycrawlers::Consumer::URL
URL
consumer class
Public Class Methods
crawlers()
click to toggle source
Returns registered crawlers
@return [Array<Crawler>]
# File lib/daimon_skycrawlers/consumer/url.rb, line 29 def crawlers @crawlers ||= [] end
queue_name()
click to toggle source
@private
# File lib/daimon_skycrawlers/consumer/url.rb, line 36 def queue_name "#{DaimonSkycrawlers.configuration.queue_name_prefix}.url" end
register(crawler)
click to toggle source
Register a given crawler
@param crawler [Crawler] crawler instance which implements `fetch` method @return [void]
# File lib/daimon_skycrawlers/consumer/url.rb, line 20 def register(crawler) crawlers << crawler end
Public Instance Methods
process(message)
click to toggle source
@private
# File lib/daimon_skycrawlers/consumer/url.rb, line 46 def process(message) crawler_interval = DaimonSkycrawlers.configuration.crawler_interval # XXX When several crawlers are registered, how should they behave? self.class.crawlers.each do |crawler| crawler.process(message.dup) if crawler.skipped? sleep(crawler_interval) if crawler.n_processed_urls % 50 == 0 else sleep(crawler_interval) end end end