class RCrawler::Async
Public Class Methods
new()
click to toggle source
# File lib/rcrawler/async.rb, line 7 def initialize @queue = ::Queue.new @config = ::RCrawler::Configuration.instance end
Public Instance Methods
crawl(&block)
click to toggle source
# File lib/rcrawler/async.rb, line 12 def crawl(&block) raise ArgumentError, "crawl method is required block" unless block_given? @queue.push block end
execute()
click to toggle source
# File lib/rcrawler/async.rb, line 17 def execute threads = [] @config.threads.times do threads << create_thread end threads.each {|thread| thread.join} end
Private Instance Methods
create_thread()
click to toggle source
# File lib/rcrawler/async.rb, line 26 def create_thread ::Thread.start do while !@queue.empty? begin Timeout::timeout(@config.timeout) {exec_crawl(@queue.pop)} rescue Timeout::Error => e raise if @config.timeout_proc == :raise end end end end
exec_crawl(block)
click to toggle source
# File lib/rcrawler/async.rb, line 38 def exec_crawl(block) crwl = Crawl.new crwl.instance_eval &block end