class Network
This class does all networking in the program. It opens network-connections, retrieves information and files from a remote site.
Constants
- MAX_FAIL
- TIMEOUT
Public Class Methods
headers(url, opt={})
click to toggle source
currently unused
# File lib/network.rb, line 84 def self::headers(url, opt={}) uri = URI.parse(url) req = Net::HTTP::Head.new(uri.path) headers = Hash.new http(uri.host, uri.port).start do |http| http.request(req) do |res| headers = res.to_hash @@log.debug('response to head-request ' << res.to_hash.to_s) end end return headers end
http(host, port)
click to toggle source
Returns a connection to 'host' via 'port', taking into account the possibility of a system-wide proxy-setting. For the time being, the proxy must be defined in the environment variable 'http_proxy', like this: http_proxy=“proxy.url:port”. Example: http_proxy=“localhost:8080” Example: http_proxy=“my.proxy.provider.com:3128”
If not defined in this way, no proxy is used and the connection will be direct.
# File lib/network.rb, line 55 def self::http(host, port) hos = nil begin hos = RbConfig['host_os'] rescue Exception => ex hos = RUBY_PLATFORM end connection = nil if(hos && /linux/ =~ hos ) proxy = ENV['http_proxy'] if(proxy) parray = proxy.split(':') if(parray.size == 3) http, phost, pport = parray if(phost.start_with?('//')) phost = phost[2..phost.size] end puts 'Using proxy at ' << phost << ', port ' << pport connection = Net::HTTP.new(host, nil, phost, pport) end end end puts 'host: ' << host << ', port: ' << port << ', direct connection, no useable proxy-setting found' if !connection connection ||= Net::HTTP.new(host,port) return connection end
http_to_file( filename, url, opts={} )
click to toggle source
Download
url, save to filename. Copied&Pasted from stackoverflow.com/questions/2263540/how-do-i-download-a-binary-file-over-http/6964173#6964173
# File lib/network.rb, line 101 def self::http_to_file( filename, url, opts={} ) opt = { :init_pause => 0.1, # start by waiting this long each time # it's deliberately long so we can see # what a full buffer looks like :learn_period => 0.3, # keep the initial pause for at least this many seconds :drop => 1.5, # fast reducing factor to find roughly optimized pause time :adjust => 1.05, # during the normal period, adjust up or down by this factor :continue => false, # by default do not retry failed downloads :previous_size => 0 # by default consider this a new file to download. }.merge(opts) @@log.debug('http_to_file, merged opts are ' << opt.inspect) pause = opt[:init_pause] learn = 1 + (opt[:learn_period]/pause).to_i drop_period = true delta = 0 max_delta = 0 continue = opt[:continue] previous_size = opt[:previous_size] length = 0 @@log.level = $log_level @@log.debug('writing to file (a)' << filename) File.open(filename.shellescape,'a') do |f| f.flock(File::LOCK_EX) # exception handling (interrupt) begin uri = URI.parse(url) @@log.debug('uri is ' << uri.to_s) @@log.debug('uri.path is ' << uri.path) # headers = headers(uri) # @@log.debug('headers is ' << headers.inspect) req = Net::HTTP::Get.new(uri.path) @@log.debug('request is ' << req.inspect) # see: https://stackoverflow.com/questions/3303029/http-range-header req['Range'] = "bytes=#{f.size}-"; @@log.debug('range will be ' << req['Range'].to_s) # begin loop, in case that continue == true count = 0 # loop num_failures = 0 begin # exception handling (network errors) begin puts "Attempt #%i) " %(count += 1) if continue # -------- begin download http(uri.host, uri.port).start do |http| bi = nil dp = nil http.request(req) do |res| @@log.debug 'response is ' << res.inspect case res when Net::HTTPSuccess # reset error-count. num_failures = 0 # if(headers.has_key?('content-Length')) if(res.key?('content-Length')) @@log.debug('content-Length is ' << (res["Content-Length"] ? res["Content-Length"] : 'N I L')) @@log.debug('file-size is: ' << (f ? f.size.to_s : 'N I L')) length = res["Content-Length"].to_i if(length == 0) puts ("No new content, file may be complete, already. Aborting") exit true elsif( f.size == 0 && previous_size == 0) @@log.debug('updating history ---->') $HISTORY.update(url, :size => length) @@log.debug('<----- did it') previous_size = length; end else @@log.warn('response without content-length') previous_size = f.size end else @@log.warn('received http response ' << res.class.name << ', code ' << res.code.to_s) @@log.error('aborting') exit false end # puts byte_units(length) puts "Fetching %s" %byte_units(length) #bi = BusyIndicator.new(true, 20) {sleep 0.4; byte_units(f.size, 3) << ' bytes of ' << byte_units($HISTORY[url].size, 3) } dp = DownloadProgress.new(f, $HISTORY[url].size) dp.start #bi = BusyIndicator.new(true, 20) {sleep 0.4; dp.progress_bar } res.read_body do |seg| f << seg $HISTORY.update(url, :position => f.size) if delta > max_delta then max_delta = delta end if learn <= 0 then learn -= 1 elsif delta == max_delta then if drop_period then pause /= opt[:drop] else pause /= opt[:adjust] end elsif delta < max_delta then drop_period = false pause *= opt[:adjust] end sleep(pause) end end if (length && f.size >= length) @@log.info('removing download from history') $HISTORY.delete(url) bi.stop('Okay') if bi dp.stop else bi.stop("Transfer interrupted prematurely at %s of %s." %[byte_units(f.size, 3), byte_units(length, 3)]) dp.stop puts "Will try to continue the download" if continue end end # ---------- end download rescue Exception => ex if ex.respond_to?(:signo) raise ex end num_failures += 1 @@log.error('An error occured while processing the response: ' << ex.message) if(num_failures < MAX_FAIL) @@log.warn('I will try to continue the download where it was interrupted') else @@log.error("#{MAX_FAIL} attempts to download have failed. Aborting for now. You should try again, later.") end end end while( (f.size < length) && continue) # end loop, if continue == true rescue Interrupt puts "\n\tProgram interrupted\n" exit 0 end end # file.open end