class RCrawler::Async

Public Class Methods

new() click to toggle source
# File lib/rcrawler/async.rb, line 7
def initialize
  @queue = ::Queue.new
  @config = ::RCrawler::Configuration.instance
end

Public Instance Methods

crawl(&block) click to toggle source
# File lib/rcrawler/async.rb, line 12
def crawl(&block)
  raise ArgumentError, "crawl method is required block" unless block_given?
  @queue.push block
end
execute() click to toggle source
# File lib/rcrawler/async.rb, line 17
def execute
  threads = []
  @config.threads.times do
    threads << create_thread
  end
  threads.each {|thread| thread.join}
end

Private Instance Methods

create_thread() click to toggle source
# File lib/rcrawler/async.rb, line 26
def create_thread
  ::Thread.start do
    while !@queue.empty?
      begin
        Timeout::timeout(@config.timeout) {exec_crawl(@queue.pop)}
      rescue Timeout::Error => e
        raise if @config.timeout_proc == :raise
      end
    end
  end
end
exec_crawl(block) click to toggle source
# File lib/rcrawler/async.rb, line 38
def exec_crawl(block)
  crwl = Crawl.new
  crwl.instance_eval &block
end