class Maxwell::Base
Public Class Methods
attr_accessor(*attrs)
click to toggle source
Calls superclass method
# File lib/maxwell.rb, line 33 def attr_accessor(*attrs) @attrs ||= [] @attrs.concat attrs super end
attrs()
click to toggle source
# File lib/maxwell.rb, line 39 def attrs @attrs || self.superclass.instance_eval("@attrs") end
concurrency(value)
click to toggle source
# File lib/maxwell.rb, line 47 def concurrency(value) @concurrency = value end
execute(args)
click to toggle source
# File lib/maxwell.rb, line 13 def execute(args) if !args[:urls].nil? urls = args[:urls] Parallel. map_with_index(urls, in_threads: @concurrency || 1) do |url, id| puts "\e[34m[#{id + 1}] scraping: #{ url }\e[0m" scraper = self.new html = Maxwell::Converter.call(url, @use_poltergeist) scraper.parser html ({ url: url }).merge(scraper.result).tap do |result_hash| scraper.handler result_hash end end else raise 'You need pass an argument urls: or raw_htmls:' end end
javascript(value)
click to toggle source
# File lib/maxwell.rb, line 43 def javascript(value) @use_poltergeist = value end
Public Instance Methods
handler(result)
click to toggle source
# File lib/maxwell.rb, line 56 def handler result p result end
parser(html)
click to toggle source
# File lib/maxwell.rb, line 52 def parser html raise NoParserDefinedErr "You need to define #{self}#parser" end
result()
click to toggle source
# File lib/maxwell.rb, line 60 def result self.class.attrs.map { |k| [k, self.send(k)] }.to_h end