class ActiveProxy::Fetcher
Attributes
cache_store[R]
options[R]
proxy_key[R]
Public Class Methods
new(proxy_key, cache_store, options = {})
click to toggle source
# File lib/active_proxy/fetcher.rb, line 7 def initialize(proxy_key, cache_store, options = {}) @proxy_key = proxy_key @cache_store = cache_store @options = options end
Public Instance Methods
current_proxy()
click to toggle source
# File lib/active_proxy/fetcher.rb, line 33 def current_proxy proxy = cache_store.fetch(cache_key("current")) do _proxy = proxy_list.sample log_message "Got new proxy #{_proxy}" _proxy end return proxy unless proxy.nil? cache_store.delete(cache_key("list")) cache_store.delete(cache_key("current")) sleep 0.5 proxy_list.sample current_proxy end
format_proxy_full()
click to toggle source
# File lib/active_proxy/fetcher.rb, line 18 def format_proxy_full proxy = current_proxy "#{proxy[:type]}://#{proxy[:address]}:#{proxy[:port]}" end
format_proxy_http()
click to toggle source
# File lib/active_proxy/fetcher.rb, line 28 def format_proxy_http proxy = current_proxy [proxy[:address], proxy[:port]] end
format_proxy_httparty()
click to toggle source
# File lib/active_proxy/fetcher.rb, line 13 def format_proxy_httparty proxy = current_proxy { http_proxyaddr: proxy[:address], http_proxyport: proxy[:port], timeout: 4 } end
format_proxy_typhoeus()
click to toggle source
# File lib/active_proxy/fetcher.rb, line 24 def format_proxy_typhoeus { proxy: format_proxy_full } end
user_agent(options = {})
click to toggle source
# File lib/active_proxy/fetcher.rb, line 48 def user_agent(options = {}) UserAgentRandomizer::UserAgent.fetch(options).string end
Private Instance Methods
cache_key(item_name)
click to toggle source
# File lib/active_proxy/fetcher.rb, line 54 def cache_key(item_name) if options[:unique_per_process] return "HTTP-PROXY-#{proxy_key}-#{Process.pid}-#{item_name}" end "HTTP-PROXY-#{proxy_key}-#{item_name}" end
call() { |self| ... }
click to toggle source
# File lib/active_proxy/fetcher.rb, line 74 def call handler = proc do |exception, attempt_number, total_delay| log_message "Retry proxy, exception: #{exception.class}" log_message exception.message[0..100] log_message "retry attempt #{attempt_number}; #{total_delay.to_i} seconds have passed." clear_current_proxy end with_retries(max_tries: max_retries_options, handler: handler) do yield self end end
clear_current_proxy()
click to toggle source
# File lib/active_proxy/fetcher.rb, line 95 def clear_current_proxy new_list = proxy_list new_list.delete(current_proxy) cache_store.delete(cache_key("current")) cache_store.write(cache_key("list"), new_list) end
log_message(msg)
click to toggle source
# File lib/active_proxy/fetcher.rb, line 102 def log_message(msg) return if ENV["LOG_ACTIVE_PROXY"].nil? p msg end
max_retries_options()
click to toggle source
# File lib/active_proxy/fetcher.rb, line 87 def max_retries_options options[:max_retries] || 10 end
proxy_list()
click to toggle source
# File lib/active_proxy/fetcher.rb, line 61 def proxy_list allowed_schemes = %w[http https] cache_store.fetch(cache_key("list")) do ProxyFetcher::Manager.new(proxy_manager_options).proxies.map do |proxy| scheme = proxy.type.downcase next unless allowed_schemes.include?(scheme) { address: proxy.addr, port: proxy.port, type: proxy.type.downcase || "http" } end.tap(&:compact!) end end
proxy_manager_options()
click to toggle source
# File lib/active_proxy/fetcher.rb, line 91 def proxy_manager_options options[:proxy_manager_options] || { filters: { maxtime: "200" } } end