class Datahen::Scraper::RubySeederExecutor
Attributes
save[RW]
Public Class Methods
exposed_methods()
click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 12 def self.exposed_methods [ :outputs, :pages, :save_pages, :save_outputs, :find_output, :find_outputs ].freeze end
new(options={})
click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 6 def initialize(options={}) @filename = options.fetch(:filename) { raise "Filename is required"} @job_id = options[:job_id] @keep_outputs = !!(options.fetch(:keep_outputs) { false }) end
Public Instance Methods
eval_seeder_script(save=false)
click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 34 def eval_seeder_script(save=false) update_seeding_starting_status proc = Proc.new do outputs = [] pages = [] begin context = isolated_binding({ outputs: outputs, pages: pages }) eval_with_context filename, context rescue Error::SafeTerminateError => e # do nothing, this is fine rescue SyntaxError => e handle_error(e) if save raise e rescue => e handle_error(e) if save raise e end puts "=========== Seeding Executed ===========" begin save_pages_and_outputs(pages, outputs, :seeding) rescue => e handle_error(e) if save raise e end update_seeding_done_status end proc.call end
exec_seeder(save=false)
click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 23 def exec_seeder(save=false) @save = save if save puts "Executing seeder script" else puts "Trying seeder script" end eval_seeder_script(save) end
handle_error(e)
click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 114 def handle_error(e) error = ["Seeding #{e.class}: #{e.to_s} (Job:#{job_id}",clean_backtrace(e.backtrace)].join("\n") seeding_update( job_id: job_id, seeding_status: :failed, log_error: error) end
save_type()
click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 70 def save_type :seeding end
update_seeding_done_status()
click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 99 def update_seeding_done_status return unless save response = seeding_update( job_id: job_id, seeding_status: :done) if response.code == 200 puts "Seeding Done." else puts "Error: Unable to save Seeding Done Status to server: #{response.body}" raise "Unable to save Seeding Done Status to server: #{response.body}" end end
update_seeding_starting_status()
click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 82 def update_seeding_starting_status return unless save response = seeding_update( job_id: job_id, seeding_status: :starting, keep_outputs: @keep_outputs ) if response.code == 200 puts "Seeding Status Updated." else puts "Error: Unable to save Seeding Status to server: #{response.body}" raise "Unable to save Seeding Status to server: #{response.body}" end end
update_to_server(opts = {})
click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 74 def update_to_server(opts = {}) seeding_update( job_id: opts[:job_id], pages: opts[:pages], outputs: opts[:outputs], seeding_status: opts[:status]) end