class Datahen::CLI::Scraper
Public Instance Methods
create(scraper_name, git_repository)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 38 def create(scraper_name, git_repository) # puts "options #{options}" client = Client::Scraper.new(options) puts "#{client.create(scraper_name, git_repository, options)}" end
delete(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 79 def delete(scraper_name) client = Client::Scraper.new(options) puts "#{client.delete(scraper_name)}" end
deploy(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 89 def deploy(scraper_name) client = Client::ScraperDeployment.new() puts "Deploying scraper. This may take a while..." puts "#{client.deploy(scraper_name)}" end
history(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 184 def history(scraper_name) client = Client::JobStat.new(options) if options[:job] puts "#{client.job_stats_history(options[:job], options)}" else puts "#{client.scraper_job_stats_history(scraper_name, options)}" end end
list()
click to toggle source
# File lib/datahen/cli/scraper.rb, line 11 def list client = Client::Scraper.new(options) puts "#{client.all}" end
log(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 123 def log(scraper_name) client = Client::JobLog.new(options) query = {} query["order"] = options.delete(:head) if options[:head] query["job_type"] = "parsing" if options[:parsing] query["job_type"] = "seeding" if options[:seeding] query["job_type"] = "finisher executing" if options[:finisher] query["page_token"] = options.delete(:more) if options[:more] query["per_page"] = options.delete(:per_page) if options[:per_page] puts "Fetching logs..." if options[:job] result = client.all_job_log(options[:job], {query: query}) else result = client.scraper_all_job_log(scraper_name, {query: query}) end if result['entries'].nil? || result["entries"].length == 0 puts "No logs yet, please try again later." else more_token = result["more_token"] result["entries"].each do |entry| puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash) end unless more_token.nil? puts "-----------" puts "To see more entries, add: \"--more #{more_token}\"" end end end
profile(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 197 def profile(scraper_name) client = Client::Scraper.new(options) puts "#{client.profile(scraper_name)}" end
show(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 73 def show(scraper_name) client = Client::Scraper.new(options) puts "#{client.find(scraper_name)}" end
start(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 105 def start(scraper_name) client = Client::ScraperJob.new(options) puts "Starting a scrape job..." puts "#{client.create(scraper_name, options)}" end
stats(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 164 def stats(scraper_name) client = Client::JobStat.new(options) if options[:job] puts "#{client.job_current_stats(options[:job], options)}" else puts "#{client.scraper_job_current_stats(scraper_name, options)}" end end
update(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper.rb, line 66 def update(scraper_name) client = Client::Scraper.new(options) puts "#{client.update(scraper_name, options)}" end