class Datahen::CLI::Scraper

Public Instance Methods

create(scraper_name, git_repository) click to toggle source
# File lib/datahen/cli/scraper.rb, line 38
def create(scraper_name, git_repository)
  # puts "options #{options}"
  client = Client::Scraper.new(options)
  puts "#{client.create(scraper_name, git_repository, options)}"
end
delete(scraper_name) click to toggle source
# File lib/datahen/cli/scraper.rb, line 79
def delete(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.delete(scraper_name)}"
end
deploy(scraper_name) click to toggle source
# File lib/datahen/cli/scraper.rb, line 89
def deploy(scraper_name)
  client = Client::ScraperDeployment.new()
  puts "Deploying scraper. This may take a while..."
  puts "#{client.deploy(scraper_name)}"
end
history(scraper_name) click to toggle source
# File lib/datahen/cli/scraper.rb, line 184
def history(scraper_name)
  client = Client::JobStat.new(options)
  if options[:job]
    puts "#{client.job_stats_history(options[:job], options)}"
  else
    puts "#{client.scraper_job_stats_history(scraper_name, options)}"
  end
end
list() click to toggle source
# File lib/datahen/cli/scraper.rb, line 11
def list
  client = Client::Scraper.new(options)
  puts "#{client.all}"
end
log(scraper_name) click to toggle source
# File lib/datahen/cli/scraper.rb, line 123
def log(scraper_name)
  client = Client::JobLog.new(options)

  query = {}
  query["order"] = options.delete(:head) if options[:head]
  query["job_type"] = "parsing" if options[:parsing]
  query["job_type"] = "seeding" if options[:seeding]
  query["job_type"] = "finisher executing" if options[:finisher]
  query["page_token"] = options.delete(:more) if options[:more]
  query["per_page"] = options.delete(:per_page) if options[:per_page]

  puts "Fetching logs..."

  if options[:job]
    result = client.all_job_log(options[:job], {query: query})
  else
    result = client.scraper_all_job_log(scraper_name, {query: query})
  end

  if result['entries'].nil? || result["entries"].length == 0
    puts "No logs yet, please try again later."
  else
    more_token = result["more_token"]

    result["entries"].each do |entry|
      puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash)
    end

    unless more_token.nil?
      puts "-----------"
      puts "To see more entries, add: \"--more #{more_token}\""
    end
  end
end
profile(scraper_name) click to toggle source
# File lib/datahen/cli/scraper.rb, line 197
def profile(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.profile(scraper_name)}"
end
show(scraper_name) click to toggle source
# File lib/datahen/cli/scraper.rb, line 73
def show(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.find(scraper_name)}"
end
start(scraper_name) click to toggle source
# File lib/datahen/cli/scraper.rb, line 105
def start(scraper_name)
  client = Client::ScraperJob.new(options)
  puts "Starting a scrape job..."
  puts "#{client.create(scraper_name, options)}"
end
stats(scraper_name) click to toggle source
# File lib/datahen/cli/scraper.rb, line 164
def stats(scraper_name)
  client = Client::JobStat.new(options)
  if options[:job]
    puts "#{client.job_current_stats(options[:job], options)}"
  else
    puts "#{client.scraper_job_current_stats(scraper_name, options)}"
  end
end
update(scraper_name) click to toggle source
# File lib/datahen/cli/scraper.rb, line 66
def update(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.update(scraper_name, options)}"
end