class Datahen::Scraper::RubySeederExecutor

Attributes

save[RW]

Public Class Methods

exposed_methods() click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 12
def self.exposed_methods
  [
    :outputs,
    :pages,
    :save_pages,
    :save_outputs,
    :find_output,
    :find_outputs
  ].freeze
end
new(options={}) click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 6
def initialize(options={})
  @filename = options.fetch(:filename) { raise "Filename is required"}
  @job_id = options[:job_id]
  @keep_outputs = !!(options.fetch(:keep_outputs) { false })
end

Public Instance Methods

eval_seeder_script(save=false) click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 34
def eval_seeder_script(save=false)
  update_seeding_starting_status

  proc = Proc.new do
    outputs = []
    pages = []

    begin
      context = isolated_binding({
        outputs: outputs,
        pages: pages
      })
      eval_with_context filename, context
    rescue Error::SafeTerminateError => e
      # do nothing, this is fine
    rescue SyntaxError => e
      handle_error(e) if save
      raise e
    rescue => e
      handle_error(e) if save
      raise e
    end

    puts "=========== Seeding Executed ==========="
    begin
      save_pages_and_outputs(pages, outputs, :seeding)
    rescue => e
      handle_error(e) if save
      raise e
    end

    update_seeding_done_status
  end
  proc.call
end
exec_seeder(save=false) click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 23
def exec_seeder(save=false)
  @save = save
  if save
    puts "Executing seeder script"
  else
    puts "Trying seeder script"
  end

  eval_seeder_script(save)
end
handle_error(e) click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 114
def handle_error(e)
  error = ["Seeding #{e.class}: #{e.to_s} (Job:#{job_id}",clean_backtrace(e.backtrace)].join("\n")

  seeding_update(
    job_id: job_id,
    seeding_status: :failed,
    log_error: error)
end
save_type() click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 70
def save_type
  :seeding
end
update_seeding_done_status() click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 99
def update_seeding_done_status
  return unless save

  response = seeding_update(
    job_id: job_id,
    seeding_status: :done)

  if response.code == 200
    puts "Seeding Done."
  else
    puts "Error: Unable to save Seeding Done Status to server: #{response.body}"
    raise "Unable to save Seeding Done Status to server: #{response.body}"
  end
end
update_seeding_starting_status() click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 82
def update_seeding_starting_status
  return unless save

  response = seeding_update(
    job_id: job_id,
    seeding_status: :starting,
    keep_outputs: @keep_outputs
  )

  if response.code == 200
    puts "Seeding Status Updated."
  else
    puts "Error: Unable to save Seeding Status to server: #{response.body}"
    raise "Unable to save Seeding Status to server: #{response.body}"
  end
end
update_to_server(opts = {}) click to toggle source
# File lib/datahen/scraper/ruby_seeder_executor.rb, line 74
def update_to_server(opts = {})
  seeding_update(
    job_id: opts[:job_id],
    pages: opts[:pages],
    outputs: opts[:outputs],
    seeding_status: opts[:status])
end