class Webgo
Constants
- MAJOR
- MINOR
- PATCH
- VERSION
Public Class Methods
call( url, headers: {} )
click to toggle source
# File lib/webget/webget.rb, line 22 def self.call( url, headers: {} ) ## assumes json format (note - encoding always utf-8 by definition! - double check?) puts " sleep #{config.sleep} sec(s)..." sleep( config.sleep ) ## slow down - sleep 3secs before each http request response = Webclient.get( url, headers: headers ) if response.status.ok? ## must be HTTP 200 puts "#{response.status.code} #{response.status.message}" ## note: use format json for pretty printing and parse check!!!! Webcache.record( url, response, format: 'json' ) else ## todo/check - log error puts "!! ERROR - #{response.status.code} #{response.status.message}:" pp response.raw ## note: dump inner (raw) response (NOT the wrapped) end ## to be done / continued response end
config()
click to toggle source
# File lib/webget/webget.rb, line 18 def self.config() @config ||= Configuration.new; end
configure() { |config| ... }
click to toggle source
lets you use
Webget.configure do |config| config.sleep = 10 end
# File lib/webget/webget.rb, line 17 def self.configure() yield( config ); end
dataset( url, encoding: 'UTF-8', headers: {} )
click to toggle source
todo/check: rename to csv or file or records or - why? why not? todo/check: rename encoding to html/http-like charset - why? why not?
# File lib/webget/webget.rb, line 91 def self.dataset( url, encoding: 'UTF-8', headers: {} ) ## assumes csv format puts " sleep #{config.sleep} sec(s)..." sleep( config.sleep ) ## slow down - sleep 3secs before each http request response = Webclient.get( url, headers: headers ) if response.status.ok? ## must be HTTP 200 puts "#{response.status.code} #{response.status.message}" Webcache.record( url, response, encoding: encoding, format: 'csv' ) ## pass along csv format - why? why not? else ## todo/check - log error puts "!! ERROR - #{response.status.code} #{response.status.message}:" pp response.raw ## note: dump inner (raw) response (NOT the wrapped) end ## to be done / continued response end
page( url, encoding: 'UTF-8', headers: {} )
click to toggle source
todo/check: rename encoding to html/http-like charset - why? why not?
# File lib/webget/webget.rb, line 44 def self.page( url, encoding: 'UTF-8', headers: {} ) ## assumes html format puts " sleep #{config.sleep} sec(s)..." sleep( config.sleep ) ## slow down - sleep 3secs before each http request response = Webclient.get( url, headers: headers ) if response.status.ok? ## must be HTTP 200 puts "#{response.status.code} #{response.status.message}" Webcache.record( url, response, encoding: encoding ) ## assumes format: html (default) else ## todo/check - log error puts "!! ERROR - #{response.status.code} #{response.status.message}:" pp response.raw ## note: dump inner (raw) response (NOT the wrapped) end ## to be done / continued response end
root()
click to toggle source
# File lib/webget/version.rb, line 18 def self.root "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}" end
text( url, path: nil, headers: {} )
click to toggle source
# File lib/webget/webget.rb, line 65 def self.text( url, path: nil, headers: {} ) ## assumes txt format puts " sleep #{config.sleep} sec(s)..." sleep( config.sleep ) ## slow down - sleep 3secs before each http request response = Webclient.get( url, headers: headers ) if response.status.ok? ## must be HTTP 200 puts "#{response.status.code} #{response.status.message}" ## note: like json assumes always utf-8 encoding for now !!! Webcache.record( url, response, path: path, ## optional "custom" (file)path for saving in cache format: 'txt' ) else ## todo/check - log error puts "!! ERROR - #{response.status.code} #{response.status.message}:" pp response.raw ## note: dump inner (raw) response (NOT the wrapped) end ## to be done / continued response end
version()
click to toggle source
# File lib/webget/version.rb, line 9 def self.version VERSION end