class Datahen::CLI::ScraperPage
Public Class Methods
Public Instance Methods
add(scraper_name, url)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 50 def add(scraper_name, url) begin options[:headers] = JSON.parse(options[:headers]) if options[:headers] options[:vars] = JSON.parse(options[:vars]) if options[:vars] method = options[:method] if options[:job] client = Client::JobPage.new(options) puts "#{client.enqueue(options[:job], method, url, options)}" else client = Client::ScraperJobPage.new(options) puts "#{client.enqueue(scraper_name, method, url, options)}" end rescue JSON::ParserError if options[:headers] puts "Error: #{options[:headers]} on headers is not a valid JSON" end if options[:vars] puts "Error: #{options[:vars]} on vars is not a valid JSON" end end end
content(scraper_name, gid)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 235 def content(scraper_name, gid) result = nil if options[:job] client = Client::JobPage.new(options) result = JSON.parse(client.find_content(options[:job], gid).to_s) else client = Client::ScraperJobPage.new(options) result = JSON.parse(client.find_content(scraper_name, gid).to_s) end if result['available'] == true puts "Preview content url: \"#{result['preview_url']}\"" begin `open "#{result['preview_url']}"` rescue end else puts "Content does not exist" end end
failedcontent(scraper_name, gid)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 258 def failedcontent(scraper_name, gid) result = nil if options[:job] client = Client::JobPage.new(options) result = JSON.parse(client.find_failed_content(options[:job], gid).to_s) else client = Client::ScraperJobPage.new(options) result = JSON.parse(client.find_failed_content(scraper_name, gid).to_s) end if result['available'] == true puts "Preview failed content url: \"#{result['preview_url']}\"" begin `open "#{result['preview_url']}"` rescue end else puts "Failed Content does not exist" end end
limbo(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 160 def limbo(scraper_name) if !options.key?(:gid) && !options.key?(:status) puts "Must specify either a --gid or --status" return end if options[:job] client = Client::JobPage.new(options) puts "#{client.limbo(options[:job])}" else client = Client::ScraperJobPage.new(options) puts "#{client.limbo(scraper_name)}" end end
list(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 21 def list(scraper_name) if options[:job] client = Client::JobPage.new(options) puts "#{client.all(options[:job])}" else client = Client::ScraperJobPage.new(options) puts "#{client.all(scraper_name)}" end end
log(scraper_name, gid)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 199 def log(scraper_name, gid) client = Client::JobLog.new(options) query = {} query["order"] = options.delete(:head) if options[:head] query["job_type"] = "parsing" if options[:parsing] query["page_token"] = options.delete(:more) if options[:more] query["per_page"] = options.delete(:per_page) if options[:per_page] puts "Fetching page logs..." if options[:job] result = client.all_job_page_log(options[:job], gid, {query: query}) else result = client.scraper_all_job_page_log(scraper_name, gid, {query: query}) end if result['entries'].nil? || result["entries"].length == 0 puts "No logs yet, please try again later." else more_token = result["more_token"] result["entries"].each do |entry| puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash) end unless more_token.nil? puts "to see more entries, add: \"--more #{more_token}\"" end end end
refetch(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 114 def refetch(scraper_name) if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type) puts "Must specify either a --gid, --fetch-fail, --parse-fail, --status or --page-type" return end if options[:job] client = Client::JobPage.new(options) puts "#{client.refetch(options[:job])}" else client = Client::ScraperJobPage.new(options) puts "#{client.refetch(scraper_name)}" end end
reparse(scraper_name)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 138 def reparse(scraper_name) if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type) puts "Must specify either a --gid, --parse-fail, --status or --page-type" return end if options[:job] client = Client::JobPage.new(options) puts "#{client.reparse(options[:job])}" else client = Client::ScraperJobPage.new(options) puts "#{client.reparse(scraper_name)}" end end
show(scraper_name, gid)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 180 def show(scraper_name, gid) if options[:job] client = Client::JobPage.new(options) puts "#{client.find(options[:job], gid)}" else client = Client::ScraperJobPage.new(options) puts "#{client.find(scraper_name, gid)}" end end
update(scraper_name, gid)
click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 85 def update(scraper_name, gid) begin options[:vars] = JSON.parse(options[:vars]) if options[:vars] if options[:job] client = Client::JobPage.new(options) puts "#{client.update(options[:job], gid, options)}" else client = Client::ScraperJobPage.new(options) puts "#{client.update(scraper_name, gid, options)}" end rescue JSON::ParserError if options[:vars] puts "Error: #{options[:vars]} on vars is not a valid JSON" end end end