class AdopsReportScrapper::GcsClient

gcs sometimes doesn't update data in 24 hours

Public Instance Methods

date_supported?(date = nil) click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 7
def date_supported?(date = nil)
  _date = date || @date
  return true if _date >= Date.today - 7
  false
end

Private Instance Methods

extract_data_from_report(site) click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 54
def extract_data_from_report(site)
  rows = @client.find_all :xpath, %Q{//table/*/tr[./td[contains(text(),"#{@date.strftime('%b')}") and contains(text(),"#{@date.strftime('%e, %Y')}")]]}
  return if rows.count == 0
  if @data.count == 0
    header = @client.find :xpath, '//table/thead/tr'
    n_header = header.find_css('td,th').map { |td| td.visible_text }
    n_header.unshift 'Site'
    @data << n_header
  end
  row = rows[0].find_css('td,th').map { |td| td.visible_text }
  row.unshift site[:name]
  @data << row
end
login() click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 15
def login
  @client.visit 'https://www.google.com/insights/consumersurveys/your-surveys'
  @client.fill_in 'Email', :with => @login
  @client.click_button 'Next'
  @client.fill_in 'Passwd', :with => @secret
  @client.click_button 'Sign in'
  # for veirfication
  # cc = @client.find :xpath, '//*[contains(text(),"the recovery phone")]'
  # cc.click
  # @client.fill_in 'Enter phone number', :with => @options[:recovery_phone]
  # @client.click_button 'Done'
  begin
    @client.find :xpath, '//*[text()="Sites"]'
  rescue Exception => e
    raise e, 'Gcs login error'
  end
end
request_report(site) click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 50
def request_report(site)
  @client.visit site[:url]
end
scrap() click to toggle source
# File lib/adops_report_scrapper/gcs_client.rb, line 33
def scrap
  sites = @client.find_all(:xpath, '//*[contains(@class,"sites-menu-item")]', visible: false)
  sites = sites.to_a
  sites.pop
  n_sites = sites.map do |site|
    {
      name: site.text(:all),
      url: site[:href].sub('settings', 'report')
    }
  end.reject { |site| site[:url].include? 'websat' }
  @data = []
  n_sites.each do |site|
    request_report site
    extract_data_from_report site
  end
end