class Blinkr::TyphoeusWrapper
Attributes
count[R]
hydra[R]
Public Class Methods
new(config, context)
click to toggle source
# File lib/blinkr/typhoeus_wrapper.rb, line 13 def initialize(config, context) @config = config.validate # Configure Typhoeus a bit Typhoeus::Config.verbose = true if config.vverbose Typhoeus::Config.cache = Blinkr::Cache.new @hydra = Typhoeus::Hydra.new(:maxconnects => (@config.maxconnects || 30), :max_total_connections => (@config.maxconnects || 30), :pipelining => false, :max_concurrency => (@config.maxconnects || 30)) Ethon.logger = Logger.new STDOUT if config.vverbose Ethon::Curl.set_option(:max_host_connections, 5, @hydra.multi.handle, :multi) @count = 0 @context = context end
Public Instance Methods
debug(url)
click to toggle source
# File lib/blinkr/typhoeus_wrapper.rb, line 39 def debug(url) process(url, @config.max_retrys) do |resp| puts "\n++++++++++" puts "+ Blinkr +" puts "++++++++++" puts "\nRequest" puts "=======" puts "Method: #{resp.request.options[:method]}" puts "Max redirects: #{resp.request.options[:maxredirs]}" puts "Follow location header: #{resp.request.options[:followlocation]}" puts "Timeout (s): #{resp.request.options[:timeout] || 'none'}" puts "Connection timeout (s): #{resp.request.options[:connecttimeout] || 'none'}" puts "\nHeaders" puts "-------" unless resp.request.options[:headers].nil? resp.request.options[:headers].each do |name, value| puts "#{name}: #{value}" end end puts "\nResponse" puts "========" puts "Status Code: #{resp.code}" puts "Status Message: #{resp.status_message}" puts "Message: #{resp.return_message}" unless resp.return_message.nil? || resp.return_message == 'No error' puts "\nHeaders" puts "-------" puts resp.response_headers end @hydra.run end
name()
click to toggle source
# File lib/blinkr/typhoeus_wrapper.rb, line 70 def name 'typhoeus' end
process(url, limit, opts = {}, &block)
click to toggle source
# File lib/blinkr/typhoeus_wrapper.rb, line 35 def process(url, limit, opts = {}, &block) _process url, limit, limit, opts, &block end
process_all(urls, limit, opts = {}, &block)
click to toggle source
# File lib/blinkr/typhoeus_wrapper.rb, line 28 def process_all(urls, limit, opts = {}, &block) urls.each do |url| process url, limit, opts, &block end @hydra.run end
Private Instance Methods
_process(url, limit, max, opts = {}, &block)
click to toggle source
# File lib/blinkr/typhoeus_wrapper.rb, line 76 def _process(url, limit, max, opts = {}, &block) unless @config.skipped? url req = Typhoeus::Request.new( url, opts.merge(:followlocation => true, :timeout => 60, :cookiefile => '_tmp/cookies', :cookiejar => '_tmp/cookies', :connecttimeout => 30, :maxredirs => 3) ) req.on_complete do |resp| if retry? resp if resp.code.to_i == 0 puts "Response code of '0', using net/http for #{url}" if @config.verbose response = nil begin uri = URI::parse url http_response = Net::HTTP.get_response(uri) response = Typhoeus::Response.new(:code => http_response.code, :status_message => http_response.message, :mock => true) rescue response = Typhoeus::Response.new(:code => 410, :status_message => 'Could not reach the resource', :mock => true) end response.request = Typhoeus::Request.new(url) Typhoeus.stub(url).and_return(response) block.call response, nil, nil next end if limit > 1 puts "Loading #{url} via typhoeus (attempt #{max - limit + 2} of #{max})" if @config.verbose _process(url, limit - 1, max, &Proc.new) else puts "Loading #{url} via typhoeus failed" if @config.verbose response = Typhoeus::Response.new(:code => 0, :status_message => "Server timed out after #{max} retries", :mock => true) response.request = Typhoeus::Request.new(url) Typhoeus.stub(url).and_return(response) block.call response, nil, nil end else block.call resp, nil, nil end end @hydra.queue req @count += 1 end end