class Datahen::CLI::Parser
Public Instance Methods
batch_exec_parse(scraper_name, config_file)
click to toggle source
# File lib/datahen/cli/parser.rb, line 76 def batch_exec_parse(scraper_name, config_file) if options[:job] job_id = options[:job] else job = Client::ScraperJob.new(options).find(scraper_name) job_id = job['id'] end # make the stdout and stderr sync to prevent buffering old_stdout_sync = $stdout.sync old_stderr_sync = $stderr.sync $stdout.sync = true $stderr.sync = true begin batch = Datahen::Scraper::BatchParser.new job_id, config_file, worker_count: options[:"workers"], max_garbage: options[:"max-garbage"], dequeue_interval: options[:"dequeue-interval"], dequeue_scale: options[:"dequeue-scale"] batch.exec_parse true, options[:"keep-outputs"] rescue => e puts [e.message] + e.backtrace end # resume whatever state the stdout and stderr sync were $stdout.sync = old_stdout_sync $stderr.sync = old_stderr_sync end
exec_parse(scraper_name, parser_file, *gids)
click to toggle source
# File lib/datahen/cli/parser.rb, line 45 def exec_parse(scraper_name, parser_file, *gids) if options[:job] job_id = options[:job] else job = Client::ScraperJob.new(options).find(scraper_name) job_id = job['id'] end gids.each do |gid| begin puts "Parsing #{gid}" vars = JSON.parse(options[:vars]) if options[:vars] puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true, vars, options[:"keep-outputs"]) rescue => e puts e end end end
try_parse(scraper_name, parser_file, gid)
click to toggle source
# File lib/datahen/cli/parser.rb, line 14 def try_parse(scraper_name, parser_file, gid) begin if options[:job] job_id = options[:job] elsif options[:global] job_id = nil else job = Client::ScraperJob.new(options).find(scraper_name) job_id = job['id'] end vars = JSON.parse(options[:vars]) if options[:vars] puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars, options[:"keep-outputs"]) rescue JSON::ParserError if options[:vars] puts "Error: #{options[:vars]} on vars is not a valid JSON" end end end