module YAVDB::Utils::HTTP
Public Class Methods
get_page_contents(url, with_cache = true, group_cache_key = 'http')
click to toggle source
# File lib/yavdb/utils/http.rb, line 25 def self.get_page_contents(url, with_cache = true, group_cache_key = 'http') puts "Requesting #{url}" if with_cache YAVDB::Utils::Cache.cache_contents(group_cache_key, url) { do_request(url) } else do_request(url) end end
Private Class Methods
do_request(url)
click to toggle source
# File lib/yavdb/utils/http.rb, line 39 def do_request(url) puts "Fetching #{url}" url = URI.parse(url) retries ||= 3 begin response = Net::HTTP.get_response(url) case response when Net::HTTPNotFound raise ArgumentError, 'page not found' when Net::HTTPTooManyRequests raise ArgumentError, 'too many requests' else response.body.lines end rescue StandardError => e raise e if retries.zero? puts "Going to retry #{url}" retries -= 1 sleep(5) retry end end