class AdopsReportScrapper::LiveintentClient

Private Instance Methods

extract_data_from_report(xlsx) click to toggle source
# File lib/adops_report_scrapper/liveintent_client.rb, line 77
def extract_data_from_report(xlsx)
  @data = xlsx.to_a.reject { |row| row[1] == '(totals)' || row[0] == '(totals)' }
end
login() click to toggle source
# File lib/adops_report_scrapper/liveintent_client.rb, line 9
def login
  @client.visit 'https://lfm.liveintent.com/'
  @client.fill_in 'username', :with => @login
  @client.fill_in 'password', :with => @secret
  @client.click_button 'Login'
  begin
    @client.find :xpath, '//*[text()="Analysis"]'
  rescue Exception => e
    raise e, 'Liveintent login error'
  end
end
request_report() click to toggle source
# File lib/adops_report_scrapper/liveintent_client.rb, line 25
def request_report
  sleep 1
  @client.find(:xpath, '//*[text()="×"]').click if @client.find_all(:xpath, '//*[text()="×"]').count > 0
  @client.find(:xpath, '//*[text()="Analysis"]').click
  @client.find(:xpath, '//*[contains(text(),"Reporting")]').click
  @client.find(:xpath, '//*[text()="New"]').click
  sleep 1
  @client.find(:xpath, '//*[text()="Publisher ID"]').click
  @client.find(:xpath, '//*[text()="Ad Slot ID"]').click
  @client.find(:xpath, '//*[text()="Add additional split"]').click
  @client.find(:xpath, '//*[text()="Year/Month"]').click
  @client.find(:xpath, '//*[text()="Device Type (inexact values)"]').click
  @client.find(:xpath, '//input[@id="intervalBegin"]').set @date.strftime('%Y-%m-%d / %Y-%m-%d')
  @client.click_button 'Download'
  wait_for_spin

  request_data = @client.driver.network_traffic.last.instance_variable_get(:@data)
  report_file_url = @client.driver.network_traffic.last.url

  cookies = @client.driver.cookies
  @client = HTTPClient.new
  @client.cookie_manager.cookies = cookies.values.map do |cookie|
    cookie = cookie.instance_variable_get(:@attributes)
    HTTP::Cookie.new cookie
  end

  header = {
    Accept: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
    Referer: 'https://lfm.liveintent.com/reporting/',
    Origin: 'https://lfm.liveintent.com',
    'User-Agent': 'Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1',
    'Content-Type': 'application/json',
  }

  @client.receive_timeout = 300
  response = @client.post(report_file_url, header: header, body: request_data['postData'] )

  tmpfile = Tempfile.new('liveintent.xlsx')
  begin
    tmpfile.binmode
    tmpfile.write(response.body)
    tmpfile.close

    xlsx = Roo::Spreadsheet.open(tmpfile.path, extension: :xlsx)
    extract_data_from_report xlsx
  ensure
    tmpfile.close
    tmpfile.unlink   # deletes the temp file
  end

end
scrap() click to toggle source
# File lib/adops_report_scrapper/liveintent_client.rb, line 21
def scrap
  request_report
end
wait_for_spin() click to toggle source
# File lib/adops_report_scrapper/liveintent_client.rb, line 81
def wait_for_spin
  30.times do |_i| # wait 5 min
    begin
      @client.find(:css, '.fa.fa-spinner.fa-spin')
    rescue Exception => e
      break
    end
    sleep 10
  end
  sleep 5
end