class Webgo

Constants

MAJOR
MINOR
PATCH
VERSION

Public Class Methods

banner() click to toggle source

version string for generator meta tag (includes ruby version)

call( url, headers: {} ) click to toggle source
# File lib/webget/webget.rb, line 22
def self.call( url, headers: {} )  ## assumes json format (note - encoding always utf-8 by definition! - double check?)
  puts "  sleep #{config.sleep} sec(s)..."
  sleep( config.sleep )   ## slow down - sleep 3secs before each http request

  response = Webclient.get( url, headers: headers )

  if response.status.ok?  ## must be HTTP 200
    puts "#{response.status.code} #{response.status.message}"
    ## note: use format json for pretty printing and parse check!!!!
    Webcache.record( url, response,
                     format: 'json' )
  else
    ## todo/check - log error
    puts "!! ERROR - #{response.status.code} #{response.status.message}:"
    pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
  end

  ## to be done / continued
  response
end
config() click to toggle source
# File lib/webget/webget.rb, line 18
def self.config()    @config ||= Configuration.new;  end
configure() { |config| ... } click to toggle source

lets you use

Webget.configure do |config|
   config.sleep = 10
end
# File lib/webget/webget.rb, line 17
def self.configure() yield( config ); end
dataset( url, encoding: 'UTF-8', headers: {} ) click to toggle source

todo/check: rename to csv or file or records or - why? why not? todo/check: rename encoding to html/http-like charset - why? why not?

# File lib/webget/webget.rb, line 91
def self.dataset( url, encoding: 'UTF-8', headers: {} )  ## assumes csv format
  puts "  sleep #{config.sleep} sec(s)..."
  sleep( config.sleep )   ## slow down - sleep 3secs before each http request

  response = Webclient.get( url, headers: headers )

  if response.status.ok?  ## must be HTTP 200
    puts "#{response.status.code} #{response.status.message}"
    Webcache.record( url, response,
                     encoding: encoding,
                     format:   'csv' )    ## pass along csv format - why? why not?
  else
    ## todo/check - log error
    puts "!! ERROR - #{response.status.code} #{response.status.message}:"
    pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
  end

  ## to be done / continued
  response
end
page( url, encoding: 'UTF-8', headers: {} ) click to toggle source

todo/check: rename encoding to html/http-like charset - why? why not?

# File lib/webget/webget.rb, line 44
def self.page( url, encoding: 'UTF-8', headers: {} )  ## assumes html format
  puts "  sleep #{config.sleep} sec(s)..."
  sleep( config.sleep )   ## slow down - sleep 3secs before each http request

  response = Webclient.get( url, headers: headers )

  if response.status.ok?  ## must be HTTP 200
    puts "#{response.status.code} #{response.status.message}"
    Webcache.record( url, response,
                     encoding: encoding  )   ## assumes format: html (default)
  else
    ## todo/check - log error
    puts "!! ERROR - #{response.status.code} #{response.status.message}:"
    pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
  end

  ## to be done / continued
  response
end
root() click to toggle source
# File lib/webget/version.rb, line 18
def self.root
  "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}"
end
text( url, path: nil, headers: {} ) click to toggle source
# File lib/webget/webget.rb, line 65
def self.text( url, path: nil, headers: {} )  ## assumes txt format
  puts "  sleep #{config.sleep} sec(s)..."
  sleep( config.sleep )   ## slow down - sleep 3secs before each http request

  response = Webclient.get( url, headers: headers )

  if response.status.ok?  ## must be HTTP 200
    puts "#{response.status.code} #{response.status.message}"
    ## note: like json assumes always utf-8 encoding for now !!!
    Webcache.record( url, response,
                     path: path,   ## optional "custom" (file)path for saving in cache
                     format: 'txt' )
  else
    ## todo/check - log error
    puts "!! ERROR - #{response.status.code} #{response.status.message}:"
    pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
  end

  ## to be done / continued
  response
end
version() click to toggle source
# File lib/webget/version.rb, line 9
def self.version
  VERSION
end