class Aquatone::Collectors::Gtr

Constants

BASE_URI
DEFAULT_PAGES_TO_PROCESS

Public Instance Methods

run() click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 17
def run
  token = nil
  pages_to_process.times do
    response = parse_response(request_page(token))
    hosts    = response.first[1].map { |a| a[1] }.uniq
    hosts.each do |host|
      add_host(host) if valid_host?(host)
    end
    _, token, _, current_page, total_pages = response.first.last
    break if token.nil? || current_page == total_pages
  end
end

Private Instance Methods

pages_to_process() click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 55
def pages_to_process
  if has_cli_option?("gtr-pages")
    return get_cli_option("gtr-pages").to_i
  end
  DEFAULT_PAGES_TO_PROCESS
end
parse_response(body) click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 44
def parse_response(body)
  body = body.split("\n", 2).last.strip
  JSON.parse(body)
end
request_page(token = nil) click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 32
def request_page(token = nil)
  if token.nil?
    uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&include_expired=true&include_subdomains=true"
  else
    uri = "#{BASE_URI}/page?domain=#{url_escape(domain.name)}&include_expired=true&include_subdomains=true&p=#{url_escape(token)}"
  end

  get_request(uri,
    { :format => :plain, :headers => { "Referer" => "https://transparencyreport.google.com/https/certificates" } }
  )
end
valid_host?(host) click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 49
def valid_host?(host)
  return false if host.start_with?("*.")
  return false unless host.end_with?(".#{domain.name}")
  true
end