class Botz::Definition
Class representing a website defined by DSL
Public Class Methods
scraper(name, options, &block)
click to toggle source
# File lib/botz/definition.rb, line 34 def scraper(name, options, &block) if options[:loop] loop_scraper(name, options, &block) else normal_scraper(name, **options, &block) end end
spider(name, start_url = nil, encoding: nil, as: :html, &block)
click to toggle source
# File lib/botz/definition.rb, line 18 def spider(name, start_url = nil, encoding: nil, as: :html, &block) connector_class = Botz::Connector.const_get(as.to_s.classify) connector = connector_class.new(start_url: start_url, encoding: encoding) spider = Botz::Spider.new(&block) spider_class = Class.new do define_singleton_method(:connector) { connector } define_singleton_method(:call) do |url = start_url, &spider_block| connector.call(url) do |resource| spider.call(resource, &spider_block) end end end const_set("#{name}_spider".classify, spider_class) spiders[name] = spider_class end
Private Class Methods
loop_scraper(name, options, &block)
click to toggle source
# File lib/botz/definition.rb, line 44 def loop_scraper(name, options, &block) options = { as: :html, start_url: nil, encoding: nil, loop: nil }.merge(options) result_class = Class.new(Botz::Result) # connector connector_class = Botz::Connector.const_get(options[:as].to_s.classify) connector = connector_class.new(encoding: options[:encoding]) namespace = Class.new do binder = Class.new(Botz::Binder) do define_singleton_method(:connector) { connector } define_singleton_method(:result_class) { result_class } define_method(:connector) { connector } instance_exec(&block) end define_singleton_method(:call) do |url = options[:start_url], &yielder| connector.call(url) do |resource| looper = Botz::Looper.new(resource, binder, options[:loop]) looper.call(&yielder) end end end const_set("#{name}_scraper".classify, namespace) scrapers[name] = namespace end
normal_scraper(name, encoding: nil, as: :html, &block)
click to toggle source
# File lib/botz/definition.rb, line 70 def normal_scraper(name, encoding: nil, as: :html, &block) # result result_class = Class.new(Botz::Result) # connector connector_class = Botz::Connector.const_get(as.to_s.classify) connector = connector_class.new(encoding: encoding) # namespace namespace = Class.new do binder = Class.new(Botz::Binder) do define_singleton_method(:connector) { connector } define_singleton_method(:result_class) { result_class } define_method(:connector) { connector } instance_exec(&block) end define_singleton_method(:bind) do |url| connector.call(url) do |resource| binder.new(resource) end end define_singleton_method(:call) do |url, &output| result = bind(url).result fail "#{url}\n#{result.errors.full_messages}" if result.invalid? output.call(result) end end const_set("#{name}_scraper".classify, namespace) scrapers[name] = namespace end
Public Instance Methods
output(&block)
click to toggle source
# File lib/botz/definition.rb, line 12 def output(&block) self.output = block end