class Network

This class does all networking in the program. It opens network-connections, retrieves information and files from a remote site.

Constants

MAX_FAIL
TIMEOUT

Public Class Methods

headers(url, opt={}) click to toggle source

currently unused

# File lib/network.rb, line 84
def self::headers(url, opt={})
        uri = URI.parse(url)
        req = Net::HTTP::Head.new(uri.path)
        headers = Hash.new
        http(uri.host, uri.port).start do |http|
                http.request(req) do |res|
                        headers = res.to_hash
                        @@log.debug('response to head-request ' << res.to_hash.to_s)
                end
        end
        return headers
end
http(host, port) click to toggle source

Returns a connection to 'host' via 'port', taking into account the possibility of a system-wide proxy-setting. For the time being, the proxy must be defined in the environment variable 'http_proxy', like this: http_proxy=“proxy.url:port”. Example: http_proxy=“localhost:8080” Example: http_proxy=“my.proxy.provider.com:3128

If not defined in this way, no proxy is used and the connection will be direct.

# File lib/network.rb, line 55
def self::http(host, port)
        hos = nil
        begin
                hos = RbConfig['host_os']
        rescue Exception => ex
                hos = RUBY_PLATFORM
        end

        connection = nil
        if(hos && /linux/ =~ hos )
                proxy = ENV['http_proxy']
                if(proxy)
                        parray = proxy.split(':')
                        if(parray.size == 3)
                                http, phost, pport = parray
                                if(phost.start_with?('//'))
                                        phost = phost[2..phost.size]
                                end
                                puts 'Using proxy at ' << phost << ', port ' << pport
                                connection = Net::HTTP.new(host, nil, phost, pport)
                        end
                end
        end
        puts 'host: ' << host << ', port: ' << port << ', direct connection, no useable proxy-setting found' if !connection
        connection ||= Net::HTTP.new(host,port)
        return connection
end
http_to_file( filename, url, opts={} ) click to toggle source

Download url, save to filename. Copied&Pasted from stackoverflow.com/questions/2263540/how-do-i-download-a-binary-file-over-http/6964173#6964173

# File lib/network.rb, line 101
def self::http_to_file( filename, url, opts={} )
        opt = {
                :init_pause => 0.1,     # start by waiting this long each time
                # it's deliberately long so we can see
                # what a full buffer looks like
                :learn_period => 0.3,   # keep the initial pause for at least this many seconds
                :drop => 1.5,           # fast reducing factor to find roughly optimized pause time
                :adjust => 1.05,        # during the normal period, adjust up or down by this factor
                :continue => false,      # by default do not retry failed downloads
                :previous_size => 0      # by default consider this a new file to download.
        }.merge(opts)

        @@log.debug('http_to_file, merged opts are ' << opt.inspect)
        pause = opt[:init_pause]
        learn = 1 + (opt[:learn_period]/pause).to_i
        drop_period = true
        delta = 0
        max_delta = 0
        continue = opt[:continue]
        previous_size = opt[:previous_size]

        length = 0
        @@log.level = $log_level 
        @@log.debug('writing to file (a)' << filename)
        File.open(filename.shellescape,'a') do |f|
                f.flock(File::LOCK_EX)
                # exception handling (interrupt)
                begin
                        uri = URI.parse(url)
                        @@log.debug('uri is ' << uri.to_s)
                        @@log.debug('uri.path is ' << uri.path)

                        # headers = headers(uri)
                        # @@log.debug('headers is ' << headers.inspect)
                
                        req = Net::HTTP::Get.new(uri.path)
                        @@log.debug('request is ' << req.inspect)
                        # see: https://stackoverflow.com/questions/3303029/http-range-header
                        req['Range'] = "bytes=#{f.size}-";
                        @@log.debug('range will be ' << req['Range'].to_s)
                        # begin loop, in case that continue == true
                        count = 0
                        # loop
                        num_failures = 0
                        begin
                                # exception handling (network errors)
                                begin 
                                        puts "Attempt #%i) " %(count += 1) if continue
                                        # -------- begin download
                                        http(uri.host, uri.port).start do |http|
                                                bi = nil
                                                dp = nil
                                                http.request(req) do |res|
                                                        @@log.debug 'response is ' << res.inspect
                                                        case res
                                                        when Net::HTTPSuccess
                                                                # reset error-count.
                                                                num_failures = 0
                                                                # if(headers.has_key?('content-Length'))
                                                                if(res.key?('content-Length'))
                                                                        @@log.debug('content-Length is ' << (res["Content-Length"] ? res["Content-Length"] : 'N I L'))
                                                                        @@log.debug('file-size is: ' << (f ? f.size.to_s : 'N I L'))
                                                                        length = res["Content-Length"].to_i
                                                                        if(length == 0)
                                                                                puts ("No new content, file may be complete, already. Aborting")
                                                                                exit true
                                                                        elsif( f.size == 0 && previous_size == 0)
                                                                                @@log.debug('updating history ---->')
                                                                                $HISTORY.update(url, :size => length)
                                                                                @@log.debug('<----- did it')
                                                                                previous_size = length;
                                                                        end
                                                                else
                                                                        @@log.warn('response without content-length')
                                                                        previous_size = f.size
                                                                end
                                                        else
                                                                @@log.warn('received http response ' << res.class.name << ', code ' << res.code.to_s)
                                                                @@log.error('aborting')
                                                                exit false
                                                        end
                                                        # puts byte_units(length)
                                                        puts "Fetching %s" %byte_units(length)
                                                        #bi = BusyIndicator.new(true, 20) {sleep 0.4; byte_units(f.size, 3) << ' bytes of ' << byte_units($HISTORY[url].size, 3)  }
                                                        dp = DownloadProgress.new(f, $HISTORY[url].size)
                                                        dp.start
                                                        #bi = BusyIndicator.new(true, 20) {sleep 0.4; dp.progress_bar  }
                                                        res.read_body do |seg|
                                                                f << seg
                                                                $HISTORY.update(url, :position => f.size)

                                                                if delta > max_delta then max_delta = delta end
                                                                if learn <= 0 then
                                                                        learn -= 1
                                                                elsif delta == max_delta then
                                                                        if drop_period then
                                                                                pause /= opt[:drop]
                                                                        else
                                                                                pause /= opt[:adjust]
                                                                        end
                                                                elsif delta < max_delta then
                                                                        drop_period = false
                                                                        pause *= opt[:adjust]
                                                                end
                                                                sleep(pause)
                                                        end
                                                end
                                                if (length && f.size >= length)
                                                        @@log.info('removing download from history')
                                                        $HISTORY.delete(url)
                                                        bi.stop('Okay') if bi
                                                        dp.stop
                                                else
                                                        bi.stop("Transfer interrupted prematurely at %s of %s." %[byte_units(f.size, 3), byte_units(length, 3)])
                                                        dp.stop
                                                        puts "Will try to continue the download" if continue
                                                end

                                        end
                                        # ---------- end download
                                rescue Exception => ex
                                        if ex.respond_to?(:signo)                                         
                                                raise ex
                                        end
                                        num_failures += 1
                                        @@log.error('An error occured while processing the response: ' << ex.message)
                                        if(num_failures < MAX_FAIL)
                                                @@log.warn('I will try to continue the download where it was interrupted')
                                        else
                                                @@log.error("#{MAX_FAIL} attempts to download have failed. Aborting for now. You should try again, later.")
                                        end
                                end
                        end while( (f.size < length) && continue)
                        # end loop, if continue == true
                rescue Interrupt 
                        puts "\n\tProgram interrupted\n"
                        exit 0
                end

        end # file.open
end