class Datahen::CLI::ScraperPage

Public Class Methods

banner(command, namespace = nil, subcommand = false) click to toggle source

Public Instance Methods

add(scraper_name, url) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 50
def add(scraper_name, url)
  begin
    options[:headers] = JSON.parse(options[:headers]) if options[:headers]
    options[:vars] = JSON.parse(options[:vars]) if options[:vars]
    method = options[:method]

    if options[:job]
      client = Client::JobPage.new(options)
      puts "#{client.enqueue(options[:job], method, url, options)}"
    else
      client = Client::ScraperJobPage.new(options)
      puts "#{client.enqueue(scraper_name, method, url, options)}"
    end

  rescue JSON::ParserError
    if options[:headers]
      puts "Error: #{options[:headers]} on headers is not a valid JSON"
    end
    if options[:vars]
      puts "Error: #{options[:vars]} on vars is not a valid JSON"
    end
  end
end
content(scraper_name, gid) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 235
def content(scraper_name, gid)
  result = nil
  if options[:job]
    client = Client::JobPage.new(options)
    result = JSON.parse(client.find_content(options[:job], gid).to_s)
  else
    client = Client::ScraperJobPage.new(options)
    result = JSON.parse(client.find_content(scraper_name, gid).to_s)
  end

  if result['available'] == true
    puts "Preview content url: \"#{result['preview_url']}\""
    begin
      `open "#{result['preview_url']}"`
    rescue
    end
  else
    puts "Content does not exist"
  end
end
failedcontent(scraper_name, gid) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 258
def failedcontent(scraper_name, gid)
  result = nil
  if options[:job]
    client = Client::JobPage.new(options)
    result = JSON.parse(client.find_failed_content(options[:job], gid).to_s)
  else
    client = Client::ScraperJobPage.new(options)
    result = JSON.parse(client.find_failed_content(scraper_name, gid).to_s)
  end

  if result['available'] == true
    puts "Preview failed content url: \"#{result['preview_url']}\""
    begin
      `open "#{result['preview_url']}"`
    rescue
    end
  else
    puts "Failed Content does not exist"
  end
end
limbo(scraper_name) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 160
def limbo(scraper_name)
  if !options.key?(:gid) && !options.key?(:status)
    puts "Must specify either a --gid or --status"
    return
  end

  if options[:job]
    client = Client::JobPage.new(options)
    puts "#{client.limbo(options[:job])}"
  else
    client = Client::ScraperJobPage.new(options)
    puts "#{client.limbo(scraper_name)}"
  end
end
list(scraper_name) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 21
def list(scraper_name)
  if options[:job]
    client = Client::JobPage.new(options)
    puts "#{client.all(options[:job])}"
  else
    client = Client::ScraperJobPage.new(options)
    puts "#{client.all(scraper_name)}"
  end
end
log(scraper_name, gid) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 199
def log(scraper_name, gid)
  client = Client::JobLog.new(options)

  query = {}
  query["order"] = options.delete(:head) if options[:head]
  query["job_type"] = "parsing" if options[:parsing]

  query["page_token"] = options.delete(:more) if options[:more]
  query["per_page"] = options.delete(:per_page) if options[:per_page]

  puts "Fetching page logs..."

  if options[:job]
    result = client.all_job_page_log(options[:job], gid, {query: query})
  else
    result = client.scraper_all_job_page_log(scraper_name, gid, {query: query})
  end

  if result['entries'].nil? || result["entries"].length == 0
    puts "No logs yet, please try again later."
  else

    more_token = result["more_token"]

    result["entries"].each do |entry|
      puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash)
    end

    unless more_token.nil?
      puts "to see more entries, add: \"--more #{more_token}\""
    end
  end
end
refetch(scraper_name) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 114
def refetch(scraper_name)
  if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
    puts "Must specify either a --gid, --fetch-fail, --parse-fail, --status or --page-type"
    return
  end

  if options[:job]
    client = Client::JobPage.new(options)
    puts "#{client.refetch(options[:job])}"
  else
    client = Client::ScraperJobPage.new(options)
    puts "#{client.refetch(scraper_name)}"
  end
end
reparse(scraper_name) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 138
def reparse(scraper_name)
  if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
    puts "Must specify either a --gid, --parse-fail, --status or --page-type"
    return
  end

  if options[:job]
    client = Client::JobPage.new(options)
    puts "#{client.reparse(options[:job])}"
  else
    client = Client::ScraperJobPage.new(options)
    puts "#{client.reparse(scraper_name)}"
  end
end
show(scraper_name, gid) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 180
def show(scraper_name, gid)
  if options[:job]
    client = Client::JobPage.new(options)
    puts "#{client.find(options[:job], gid)}"
  else
    client = Client::ScraperJobPage.new(options)
    puts "#{client.find(scraper_name, gid)}"
  end
end
update(scraper_name, gid) click to toggle source
# File lib/datahen/cli/scraper_page.rb, line 85
def update(scraper_name, gid)
  begin
    options[:vars] = JSON.parse(options[:vars]) if options[:vars]

    if options[:job]
      client = Client::JobPage.new(options)
      puts "#{client.update(options[:job], gid, options)}"
    else
      client = Client::ScraperJobPage.new(options)
      puts "#{client.update(scraper_name, gid, options)}"
    end

  rescue JSON::ParserError
    if options[:vars]
      puts "Error: #{options[:vars]} on vars is not a valid JSON"
    end
  end
end