class ProxyFetcher::Manager

ProxyFetcher Manager class for interacting with proxy lists from various providers.

Constants

REFRESHER_LOCK

Attributes

proxies[R]

@!attribute [r] proxies

@return [Array<ProxyFetcher::Proxy>] An array of proxies

Public Class Methods

from_file(files, **options)
Alias for: from_files
from_files(files, **options) click to toggle source
# File lib/proxy_fetcher/manager.rb, line 9
def from_files(files, **options)
  new(**options.merge(files: Array(files)))
end
Also aliased as: from_file
new(**options) click to toggle source

Initialize ProxyFetcher Manager instance for managing proxies

refresh: true - load proxy list from the remote server on initialization refresh: false - just initialize the class, proxy list will be empty ([])

@return [Manager]

# File lib/proxy_fetcher/manager.rb, line 27
def initialize(**options)
  if options.fetch(:refresh, true)
    refresh_list!(options.fetch(:filters, {}))
  else
    @proxies = []
  end

  files = Array(options.fetch(:file, options.fetch(:files, [])))
  load_proxies_from_files!(files) if files&.any?

  cleanup! if options.fetch(:validate, false)
end

Public Instance Methods

cleanup!() click to toggle source

Clean current proxy list from dead proxies (that doesn't respond by timeout)

@return [Array<ProxyFetcher::Proxy>]

list of valid proxies
# File lib/proxy_fetcher/manager.rb, line 129
def cleanup!
  valid_proxies = ProxyListValidator.new(@proxies).validate
  @proxies &= valid_proxies
end
Also aliased as: validate!
fetch!(filters = nil)
Alias for: refresh_list!
get() click to toggle source

Pop just first proxy (and back it to the end of the proxy list).

@return [ProxyFetcher::Proxy, NilClass]

proxy object from the list
# File lib/proxy_fetcher/manager.rb, line 74
def get
  return if @proxies.empty?

  first_proxy = @proxies.shift
  @proxies << first_proxy

  first_proxy
end
Also aliased as: pop
get!() click to toggle source

Pop first valid proxy (and back it to the end of the proxy list) Invalid proxies will be removed from the list

@return [ProxyFetcher::Proxy, NilClass]

proxy object from the list
# File lib/proxy_fetcher/manager.rb, line 91
def get!
  index = proxies.find_index(&:connectable?)
  return if index.nil?

  proxy = proxies.delete_at(index)
  tail = proxies[index..-1]

  @proxies = tail << proxy

  proxy
end
Also aliased as: pop!
inspect() click to toggle source

@private No need to put all the attr_readers to the output

# File lib/proxy_fetcher/manager.rb, line 157
def inspect
  to_s
end
load_proxies_from_files!(proxy_files) click to toggle source

Loads proxies from files.

@param proxy_files [String, Array<String,Pathname>]

file path of list of files to load
# File lib/proxy_fetcher/manager.rb, line 110
def load_proxies_from_files!(proxy_files)
  proxy_files = Array(proxy_files)
  return if proxy_files.empty?

  proxy_files.each do |proxy_file|
    File.foreach(proxy_file, chomp: true) do |proxy_string|
      addr, port = proxy_string.split(":", 2)
      port = Integer(port) if port
      @proxies << Proxy.new(addr: addr, port: port)
    end
  end

  @proxies.uniq!
end
pop()
Alias for: get
pop!()
Alias for: get!
random()
Alias for: random_proxy
random_proxy() click to toggle source

Returns random proxy

@return [Proxy]

random proxy from the loaded list
# File lib/proxy_fetcher/manager.rb, line 141
def random_proxy
  proxies.sample
end
Also aliased as: random
raw_proxies() click to toggle source

Returns array of proxy URLs (just schema + host + port)

@return [Array<String>]

collection of proxies
# File lib/proxy_fetcher/manager.rb, line 152
def raw_proxies
  proxies.map(&:url)
end
refresh_list!(filters = nil) click to toggle source

Update current proxy list using configured providers.

@param filters [Hash] providers filters

# File lib/proxy_fetcher/manager.rb, line 44
def refresh_list!(filters = nil)
  @proxies = []
  threads = []

  ProxyFetcher.config.providers.each do |provider_name|
    threads << Thread.new do
      Thread.current.report_on_exception = false

      provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name)
      provider_filters = filters && filters.fetch(provider_name.to_sym, filters)
      provider_proxies = provider.fetch_proxies!(provider_filters)

      REFRESHER_LOCK.synchronize do
        @proxies.concat(provider_proxies)
      end
    end
  end

  threads.each(&:join)

  @proxies
end
Also aliased as: fetch!
validate!()
Alias for: cleanup!