class Aquatone::Collectors::Gtr
Constants
- BASE_URI
- DEFAULT_PAGES_TO_PROCESS
Public Instance Methods
run()
click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 17 def run token = nil pages_to_process.times do response = parse_response(request_page(token)) hosts = response.first[1].map { |a| a[1] }.uniq hosts.each do |host| add_host(host) if valid_host?(host) end _, token, _, current_page, total_pages = response.first.last break if token.nil? || current_page == total_pages end end
Private Instance Methods
pages_to_process()
click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 55 def pages_to_process if has_cli_option?("gtr-pages") return get_cli_option("gtr-pages").to_i end DEFAULT_PAGES_TO_PROCESS end
parse_response(body)
click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 44 def parse_response(body) body = body.split("\n", 2).last.strip JSON.parse(body) end
request_page(token = nil)
click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 32 def request_page(token = nil) if token.nil? uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&include_expired=true&include_subdomains=true" else uri = "#{BASE_URI}/page?domain=#{url_escape(domain.name)}&include_expired=true&include_subdomains=true&p=#{url_escape(token)}" end get_request(uri, { :format => :plain, :headers => { "Referer" => "https://transparencyreport.google.com/https/certificates" } } ) end
valid_host?(host)
click to toggle source
# File lib/aquatone/collectors/gtr.rb, line 49 def valid_host?(host) return false if host.start_with?("*.") return false unless host.end_with?(".#{domain.name}") true end