class Botz::Shell
botz shell interface
Attributes
definition_file[R]
Public Class Methods
new(definition_file)
click to toggle source
# File lib/botz/shell.rb, line 10 def initialize(definition_file) @definition_file = definition_file end
Public Instance Methods
build(name)
click to toggle source
rubocop:disable Metrics/MethodLength
# File lib/botz/shell.rb, line 54 def build(name) File.open("#{name}.rb", 'w') do |f| f.write <<~RUBY # frozen_string_literal: true Botz.define(:#{name}) do spider(:example, 'http://example.com') do |html, yielder| # yielder.call(url or resource) end scraper(:example) do end end RUBY end File.open("#{name}.sh", 'w') do |f| f.write <<~SHELL #!/bin/bash eval "$(botz $(dirname "${0}")/#{name}.rb shell)" spider example SHELL end end
function()
click to toggle source
# File lib/botz/shell.rb, line 42 def function print <<~SHELL function spider() { botz spider #{definition_file.path} $1 } function scraper() { botz scraper #{definition_file.path} $1 } SHELL end
scraper(name)
click to toggle source
rubocop:disable Lint/AssignmentInCondition, Style/RescueStandardError
# File lib/botz/shell.rb, line 15 def scraper(name) command = scrapers[name.to_sym] fail "undefined commmand[#{name}]" if command.nil? while line = STDIN.gets url = line.strip begin command.call(url, &definition_file.output) rescue => e STDERR.puts "ERROR #{url}: #{e}\n#{e.backtrace}" end end end
spider(name)
click to toggle source
rubocop:enable Lint/AssignmentInCondition, Style/RescueStandardError
# File lib/botz/shell.rb, line 30 def spider(name) command = spiders[name.to_sym] if File.pipe?(STDIN) STDIN.each_line do |line| start_url = line.strip command.call(start_url) { |url| puts url } end else command.call { |url| puts url } end end