class ActiveProxy::Fetcher

Attributes

cache_store[R]
options[R]
proxy_key[R]

Public Class Methods

new(proxy_key, cache_store, options = {}) click to toggle source
# File lib/active_proxy/fetcher.rb, line 7
def initialize(proxy_key, cache_store, options = {})
  @proxy_key = proxy_key
  @cache_store = cache_store
  @options = options
end

Public Instance Methods

current_proxy() click to toggle source
# File lib/active_proxy/fetcher.rb, line 33
def current_proxy
  proxy = cache_store.fetch(cache_key("current")) do
    _proxy = proxy_list.sample
    log_message "Got new proxy #{_proxy}"
    _proxy
  end
  return proxy unless proxy.nil?
  cache_store.delete(cache_key("list"))
  cache_store.delete(cache_key("current"))
  sleep 0.5
  proxy_list.sample

  current_proxy
end
format_proxy_full() click to toggle source
# File lib/active_proxy/fetcher.rb, line 18
def format_proxy_full
  proxy = current_proxy

  "#{proxy[:type]}://#{proxy[:address]}:#{proxy[:port]}"
end
format_proxy_http() click to toggle source
# File lib/active_proxy/fetcher.rb, line 28
def format_proxy_http
  proxy = current_proxy
  [proxy[:address], proxy[:port]]
end
format_proxy_httparty() click to toggle source
# File lib/active_proxy/fetcher.rb, line 13
def format_proxy_httparty
  proxy = current_proxy
  { http_proxyaddr: proxy[:address], http_proxyport: proxy[:port], timeout: 4 }
end
format_proxy_typhoeus() click to toggle source
# File lib/active_proxy/fetcher.rb, line 24
def format_proxy_typhoeus
  { proxy: format_proxy_full }
end
user_agent(options = {}) click to toggle source
# File lib/active_proxy/fetcher.rb, line 48
def user_agent(options = {})
  UserAgentRandomizer::UserAgent.fetch(options).string
end

Private Instance Methods

cache_key(item_name) click to toggle source
# File lib/active_proxy/fetcher.rb, line 54
def cache_key(item_name)
  if options[:unique_per_process]
    return "HTTP-PROXY-#{proxy_key}-#{Process.pid}-#{item_name}"
  end
  "HTTP-PROXY-#{proxy_key}-#{item_name}"
end
call() { |self| ... } click to toggle source
# File lib/active_proxy/fetcher.rb, line 74
def call
  handler = proc do |exception, attempt_number, total_delay|
    log_message "Retry proxy, exception: #{exception.class}"
    log_message exception.message[0..100]
    log_message "retry attempt #{attempt_number}; #{total_delay.to_i} seconds have passed."
    clear_current_proxy
  end

  with_retries(max_tries: max_retries_options, handler: handler) do
    yield self
  end
end
clear_current_proxy() click to toggle source
# File lib/active_proxy/fetcher.rb, line 95
def clear_current_proxy
  new_list = proxy_list
  new_list.delete(current_proxy)
  cache_store.delete(cache_key("current"))
  cache_store.write(cache_key("list"), new_list)
end
log_message(msg) click to toggle source
# File lib/active_proxy/fetcher.rb, line 102
def log_message(msg)
  return if ENV["LOG_ACTIVE_PROXY"].nil?
  p msg
end
max_retries_options() click to toggle source
# File lib/active_proxy/fetcher.rb, line 87
def max_retries_options
  options[:max_retries] || 10
end
proxy_list() click to toggle source
# File lib/active_proxy/fetcher.rb, line 61
def proxy_list
  allowed_schemes = %w[http https]

  cache_store.fetch(cache_key("list")) do
    ProxyFetcher::Manager.new(proxy_manager_options).proxies.map do |proxy|
      scheme = proxy.type.downcase
      next unless allowed_schemes.include?(scheme)

      { address: proxy.addr, port: proxy.port, type: proxy.type.downcase || "http" }
    end.tap(&:compact!)
  end
end
proxy_manager_options() click to toggle source
# File lib/active_proxy/fetcher.rb, line 91
def proxy_manager_options
  options[:proxy_manager_options] || { filters: { maxtime: "200" } }
end