class AeEasy::Qa::ValidateCollection
Attributes
collection_name[R]
errors[R]
options[R]
outputs[R]
rules[R]
scraper_name[R]
total_records[R]
Public Class Methods
new(scraper_name, collection_name, total_records, rules, outputs, options)
click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 107 def initialize(scraper_name, collection_name, total_records, rules, outputs, options) @scraper_name = scraper_name @collection_name = collection_name @total_records = total_records @rules = rules @outputs = outputs @data = options['data'] @options = options @errors = { errored_items: [] } end
Public Instance Methods
run()
click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 118 def run output_collection if most_recent_finished_job puts "data count #{data.count}" if data.any? ValidateGroups.new(data, scraper_name, collection_name, errors).run ValidateRules.new(data, errors, rules).run if rules end SaveOutput.new(data.count, rules, errors, outputs_collection_name, outputs, options).run else puts "No job with status 'done' available" end end
Private Instance Methods
data()
click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 149 def data @data ||= begin data = [] page = 1 while data.count < total_records records = AnswersEngine::Client::JobOutput.new(per_page:500, page: page).all(most_recent_finished_job['id'], collection_name).parsed_response sleep 1 if records records.each do |record| data << record end else puts "All ScraperJobOutput request was nil. Total records: #{total_records}, data count: #{data.count}, page: #{page}" break end page += 1 end data end end
most_recent_finished_job()
click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 138 def most_recent_finished_job @most_recent_finished_job ||= begin jobs_response = AnswersEngine::Client::ScraperJob.new.all(scraper_name) if jobs_response.code == 200 jobs_response.parsed_response.sort_by { |job| job['created_at'] }.reverse.find{|job| job['status'] == 'active' || job['status'] == 'done' } else nil end end end
output_collection()
click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 134 def output_collection puts "Validating collection: #{collection_name}" end
outputs_collection_name()
click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 170 def outputs_collection_name @outputs_collection_name ||= "#{scraper_name}_#{collection_name}" end