class AdopsReportScrapper::GcsClient
gcs sometimes doesn't update data in 24 hours
Public Instance Methods
date_supported?(date = nil)
click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 7 def date_supported?(date = nil) _date = date || @date return true if _date >= Date.today - 7 false end
Private Instance Methods
extract_data_from_report(site)
click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 54 def extract_data_from_report(site) rows = @client.find_all :xpath, %Q{//table/*/tr[./td[contains(text(),"#{@date.strftime('%b')}") and contains(text(),"#{@date.strftime('%e, %Y')}")]]} return if rows.count == 0 if @data.count == 0 header = @client.find :xpath, '//table/thead/tr' n_header = header.find_css('td,th').map { |td| td.visible_text } n_header.unshift 'Site' @data << n_header end row = rows[0].find_css('td,th').map { |td| td.visible_text } row.unshift site[:name] @data << row end
login()
click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 15 def login @client.visit 'https://www.google.com/insights/consumersurveys/your-surveys' @client.fill_in 'Email', :with => @login @client.click_button 'Next' @client.fill_in 'Passwd', :with => @secret @client.click_button 'Sign in' # for veirfication # cc = @client.find :xpath, '//*[contains(text(),"the recovery phone")]' # cc.click # @client.fill_in 'Enter phone number', :with => @options[:recovery_phone] # @client.click_button 'Done' begin @client.find :xpath, '//*[text()="Sites"]' rescue Exception => e raise e, 'Gcs login error' end end
request_report(site)
click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 50 def request_report(site) @client.visit site[:url] end
scrap()
click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 33 def scrap sites = @client.find_all(:xpath, '//*[contains(@class,"sites-menu-item")]', visible: false) sites = sites.to_a sites.pop n_sites = sites.map do |site| { name: site.text(:all), url: site[:href].sub('settings', 'report') } end.reject { |site| site[:url].include? 'websat' } @data = [] n_sites.each do |site| request_report site extract_data_from_report site end end