class Scraypa::UserAgentIterator

Attributes

current_user_agent[R]

Public Class Methods

new(*args) click to toggle source
Calls superclass method Scraypa::UserAgentAbstract::new
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 5
def initialize *args
  super(*args)
  @config = args[0] || {}
  @change_after_n_requests = @config.fetch(:change_after_n_requests, 0)
  @list_limit = @config.fetch(:list_limit, 0).to_i
  @strategy = @config.fetch(:strategy, :roundrobin)
  @list = limit_list to_array@config.fetch(:list, USER_AGENT_LIST)
  @reducing_list = @list.clone
  @current_user_agent = nil
  @current_user_agent_requests = 0
end

Public Instance Methods

list() click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 27
def list
  @list
end
user_agent() click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 17
def user_agent
  get_a_new_user_agent? ? (
    @current_user_agent_requests = 0
    select_user_agent_using_strategy
  ) : (
    @current_user_agent_requests += 1
    @current_user_agent
  )
end

Private Instance Methods

ensure_a_new_random_user_agent() click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 81
def ensure_a_new_random_user_agent
  return @list.first if @list.length == 1
  random_user_agent = nil
  loop do
    random_user_agent = @list.sample
    break unless random_user_agent == @current_user_agent
  end
  random_user_agent
end
get_a_new_user_agent?() click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 65
def get_a_new_user_agent?
  !@current_user_agent ||
      @current_user_agent_requests >= @change_after_n_requests
end
limit_list(list) click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 44
def limit_list list
  @list_limit <= 0 || @list_limit >= list.length ?
      list :
      @strategy == :randomize ?
          limit_list_randomly(list) :
          list[0..@list_limit-1]
end
limit_list_randomly(list) click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 52
def limit_list_randomly list
  random_list = []
  loop do
    sample = list.sample
    if list.include? sample
      random_list << sample
      list.delete(sample)
    end
    break if random_list.length >= @list_limit
  end
  random_list
end
next_user_agent_from_list() click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 91
def next_user_agent_from_list
  @reducing_list = @list.clone if @reducing_list.empty?
  @current_user_agent_requests += 1
  @current_user_agent = @reducing_list.shift
end
random_user_agent_from_list() click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 76
def random_user_agent_from_list
  @current_user_agent_requests += 1
  @current_user_agent = ensure_a_new_random_user_agent
end
select_user_agent_using_strategy() click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 70
def select_user_agent_using_strategy
  @strategy == :randomize ?
      random_user_agent_from_list :
      next_user_agent_from_list
end
to_array(variable) click to toggle source
# File lib/scraypa/user_agent/user_agent_iterator.rb, line 33
def to_array variable
  case variable
    when Array
      variable
    when Hash
      variable.values
    else
      [variable]
  end
end