module YAVDB::Utils::HTTP

Public Class Methods

get_page_contents(url, with_cache = true, group_cache_key = 'http') click to toggle source
# File lib/yavdb/utils/http.rb, line 25
def self.get_page_contents(url, with_cache = true, group_cache_key = 'http')
  puts "Requesting #{url}"

  if with_cache
    YAVDB::Utils::Cache.cache_contents(group_cache_key, url) { do_request(url) }
  else
    do_request(url)
  end
end

Private Class Methods

do_request(url) click to toggle source
# File lib/yavdb/utils/http.rb, line 39
def do_request(url)
  puts "Fetching #{url}"

  url = URI.parse(url)
  retries ||= 3

  begin
    response = Net::HTTP.get_response(url)
    case response
      when Net::HTTPNotFound
        raise ArgumentError, 'page not found'
      when Net::HTTPTooManyRequests
        raise ArgumentError, 'too many requests'
      else
        response.body.lines
    end
  rescue StandardError => e
    raise e if retries.zero?

    puts "Going to retry #{url}"
    retries -= 1
    sleep(5)
    retry
  end
end