module DaimonSkycrawlers

configuration() click to toggle source

Retrieve configuration object

@return [DaimonSkycrawlers::Configuration]

# File lib/daimon_skycrawlers.rb, line 52
def configuration
  @configuration ||= DaimonSkycrawlers::Configuration.new.tap do |config|
    config.logger = DaimonSkycrawlers::Logger.default
    config.queue_name_prefix = "daimon-skycrawlers"
    config.crawler_interval = 1
    config.shutdown_interval = 10
  end
end

configure() { |configuration| ... } click to toggle source

Configure DaimonSkycrawlers

“`ruby DaimonSkycrawlers.configure do |config|

config.logger = DaimonSkycrawlers::Logger.default
config.queue_name_prefix = "daimon-skycrawlers"
config.crawler_interval = 1
config.shutdown_interval = 10

end “`

logger: logger instance
queue_name_prefix: prefix of queue name.
crawler_interval: crawling interval
shutdown_interval: shutdown after interval after the queue is empty

@return [void] @yield [configuration] configure DaimonSkycrawlers @yieldparam configuration [DaimonSkycrawlers::Configuration] configuration object @yieldreturn [void]

# File lib/daimon_skycrawlers.rb, line 82
def configure
  yield configuration
end

env() click to toggle source

Return current environment

# File lib/daimon_skycrawlers.rb, line 125
def env
  ENV["SKYCRAWLERS_ENV"] || "development"
end

load_crawlers() click to toggle source

Load “app/crawlers/*/.rb”

@return [void]

# File lib/daimon_skycrawlers.rb, line 103
def load_crawlers
  Dir.glob("app/crawlers/**/*.rb") do |path|
    require(File.expand_path(path, Dir.pwd)) &&
      DaimonSkycrawlers.configuration.logger.info("Loaded crawler: #{path}")
  end
end

load_init() click to toggle source

Load “config/init.rb”

@return [void]

# File lib/daimon_skycrawlers.rb, line 91
def load_init
  require(File.expand_path("config/init.rb", Dir.pwd))
rescue LoadError => ex
  puts ex.message
  exit(false)
end

load_processors() click to toggle source

Load “app/processors/*/.rb”

@return [void]

# File lib/daimon_skycrawlers.rb, line 115
def load_processors
  Dir.glob("app/processors/**/*.rb") do |path|
    require(File.expand_path(path, Dir.pwd)) &&
      DaimonSkycrawlers.configuration.logger.info("Loaded processor: #{path}")
  end
end

register_crawler(crawler) click to toggle source

Register a crawler

@param crawler [Crawler] instance which implements `fetch` method @return [void]

# File lib/daimon_skycrawlers.rb, line 43
def register_crawler(crawler)
  DaimonSkycrawlers::Consumer::URL.register(crawler)
end

register_processor(processor = nil, &block) click to toggle source

Register a processor

@overload register_processor(processor)

@param processor [Processor] instance which implements `call` method
@return [void]

@overload register_processor

@return [void]
@yield [message] Register given block as a processor.
@yieldparam message [Hash] A message from queue
@yieldreturn [void]

# File lib/daimon_skycrawlers.rb, line 33
def register_processor(processor = nil, &block)
  DaimonSkycrawlers::Consumer::HTTPResponse.register(processor, &block)
end

Constants

Public Class Methods