module PageByPage::Common
Public Class Methods
new(opt = {}, &block)
click to toggle source
# File lib/page_by_page/common.rb, line 6 def initialize(opt = {}, &block) @progress = {} opt.each{ |name, value| send name, value } instance_eval &block if block end
Public Instance Methods
header(hash)
click to toggle source
# File lib/page_by_page/common.rb, line 20 def header hash @header = hash end
interval(second)
click to toggle source
# File lib/page_by_page/common.rb, line 24 def interval second @interval = second end
no_progress(*arg)
click to toggle source
# File lib/page_by_page/common.rb, line 28 def no_progress *arg @progress = nil end
selector(sl)
click to toggle source
# File lib/page_by_page/common.rb, line 16 def selector sl @selector = sl end
to(n)
click to toggle source
# File lib/page_by_page/common.rb, line 12 def to n @to = n end
Protected Instance Methods
http_header()
click to toggle source
# File lib/page_by_page/common.rb, line 46 def http_header @http_header ||= ( h = {} Hash(@header).each_pair{ |k, v| h[k.to_s] = v } h ) end
limit()
click to toggle source
# File lib/page_by_page/common.rb, line 54 def limit @to ||= Float::INFINITY end
parse(url)
click to toggle source
# File lib/page_by_page/common.rb, line 34 def parse url url = URI::encode url page = open(url, http_header) Nokogiri::HTML page.read rescue OpenURI::HTTPError => e if e.message == '404 Not Found' throw :no_more else raise e end end
update_progress(thread, page_num)
click to toggle source
# File lib/page_by_page/common.rb, line 58 def update_progress thread, page_num @progress[thread] = page_num printf "\r%s => %s", Time.now.strftime('%F %T'), @progress.values.sort end