module DaimonSkycrawlers::Crawler

Name space for crawler related classes

Public Class Methods

config() click to toggle source

Shortcut of DaimonSkycrawlers.configuration

@return [DaimonSkycrawlers::Configuration]

# File lib/daimon_skycrawlers/crawler.rb, line 41
def config
  DaimonSkycrawlers.configuration
end
default_process_name() click to toggle source

Default process name

@return [String] Default process name

# File lib/daimon_skycrawlers/crawler.rb, line 59
def default_process_name
  "#{config.queue_name_prefix}:url"
end
enqueue_url(url, message = {}) click to toggle source

Enqueue a URL to crawler queue

@param url [String] Specify absolute URL @param message [Hash] Extra parameters for crawler @return [void]

# File lib/daimon_skycrawlers/crawler.rb, line 30
def enqueue_url(url, message = {})
  message[:url] = url
  config.logger.debug("#{queue_name}: #{url}")
  SongkickQueue.publish(queue_name, message)
end
queue_name() click to toggle source

Queue name for crawler

@return [String] Queue name

# File lib/daimon_skycrawlers/crawler.rb, line 50
def queue_name
  "#{config.queue_name_prefix}.url"
end
run(process_name: default_process_name) click to toggle source

Run registered crawlers

@param process_name [String] Process name

# File lib/daimon_skycrawlers/crawler.rb, line 17
def run(process_name: default_process_name)
  if config.shutdown_interval > 0
    DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
  end
  SongkickQueue::Worker.new(process_name, [DaimonSkycrawlers::Consumer::URL]).run
end