class Requester

Public Class Methods

new(options = {}) click to toggle source
# File lib/reid/requester.rb, line 5
def initialize(options = {})
        @min_request_interval=(options[:min_request_interval] || false)
        @error_log = (options[:error_log] || false)
        @max_backoff_time = (options[:max_backoff_time] || false)
        @initial_delay = (options[:intial_delay] || 1.0)
        @multiplicand = (options[:multiplicand] || 1.3)
        @previous_request = (@min_request_interval && Time.now)
end

Public Instance Methods

damper() click to toggle source
# File lib/reid/requester.rb, line 37
def damper()
        if    (Time.now - @previous_request < @min_request_interval)
                sleep(@min_request_interval - (Time.now - @previous_request))
        end
        @previous_request = Time.now
end
request(url, time = false) click to toggle source
# File lib/reid/requester.rb, line 14
def request(url, time = false)
        if !time then if @min_request_interval then damper end end
        begin
                return Nokogiri::HTML(open(url))
        rescue Exception => e
                if @error_record
                        error_record = {:time => Time.now, :message =>e.message}
                        @error_log.insert(error_record)
                end
                unless time
                        sleep @intial_delay
                        return request url, @initial_delay * @multiplicand
                else
                        if (!@max_backoff_time || (time < @max_backoff_time))
                                sleep time
                                return request url, time * @multiplicand
                        else
                                raise "Problem with request. Max backoff time exceided."
                        end
                end
        end
end