module Intrigue::Ident
Constants
- VERSION
Public Instance Methods
check_intrigue_uri_hash(intrigue_uri_data, options={})
click to toggle source
# File lib/intrigue-ident.rb, line 58 def check_intrigue_uri_hash(intrigue_uri_data, options={}) results = [] # gather all fingeprints for each product # this will look like an array of checks, each with a uri and a SET of checks generated_checks = Intrigue::Ident::CheckFactory.all.map{|x| x.new.generate_checks("x") }.flatten # group by the uris, with the associated checks # TODO - this only currently supports the first path of the group!!!! grouped_generated_checks = generated_checks.group_by{|x| x[:paths].first } # call the check on each uri grouped_generated_checks.each do |ggc| target_url = ggc.first # call each check, collecting the product if it's a match ggc.last.each do |check| results << _match_uri_hash(check, intrigue_uri_data, options) end end # Return all matches, minus the nils (non-matches) results.compact end
generate_requests_and_check(url, options)
click to toggle source
# File lib/intrigue-ident.rb, line 20 def generate_requests_and_check(url, options) results = [] # gather all fingeprints for each product # this will look like an array of checks, each with a uri and a SET of checks generated_checks = Intrigue::Ident::CheckFactory.all.map{|x| x.new.generate_checks(url) }.flatten # group by the uris, with the associated checks # TODO - this only currently supports the first path of the group!!!! grouped_generated_checks = generated_checks.group_by{|x| x[:paths].first } # call the check on each uri grouped_generated_checks.each do |ggc| target_url = ggc.first # get the response response = _http_request :get, "#{target_url}" unless response puts "Unable to get a response at: #{target_url}, failing" return nil end # Go ahead and match it up if we got a response! if response # call each check, collecting the product if it's a match ggc.last.each do |check| results << _match_http_response(check, response, options) end end end # Return all matches, minus the nils (non-matches) results.compact end
remove_bad_ident_matches(matches)
click to toggle source
remove bad checks we need to roll back
# File lib/intrigue-ident.rb, line 86 def remove_bad_ident_matches(matches) passed_matches = [] matches.each do |m| next if (m["match_type"] == "content_body" && m["matched_content"] == "(?-mix:Drupal)") next if (m["match_type"] == "content_cookies" && m["matched_content"] == "(?i-mx:ADRUM_BTa)" && m["product"] == "Jobvite") passed_matches << m end passed_matches end
Private Instance Methods
_construct_match_response(check, data, options={})
click to toggle source
# File lib/intrigue-ident.rb, line 103 def _construct_match_response(check, data, options={}) calculated_version = (check[:dynamic_version].call(data) if check[:dynamic_version]) || check[:version] calculated_type = "a" if check[:type] == "application" calculated_type = "h" if check[:type] == "hardware" calculated_type = "o" if check[:type] == "operating_system" calculated_type = "s" if check[:type] == "service" # literally made up cpe_string = "cpe:2.3:#{calculated_type}:#{check[:vendor]}:#{check[:product]}".downcase cpe_string << ":#{calculated_version}".downcase if calculated_version to_return = { "type" => check[:type], "vendor" => check[:vendor], "product" => check[:product], "version" => calculated_version, "tags" => check[:tags], "matched_content" => check[:match_content], "match_type" => check[:match_type], "match_details" => check[:match_details], "hide" => check[:hide], "cpe" => cpe_string, } if options[:match_vulns] to_return["vulns"] = Cpe.new(cpe_string).vulns end to_return end
_http_request(method, uri_string, credentials=nil, headers={}, data=nil, limit = 10, open_timeout=15, read_timeout=15)
click to toggle source
# File lib/intrigue-ident.rb, line 249 def _http_request(method, uri_string, credentials=nil, headers={}, data=nil, limit = 10, open_timeout=15, read_timeout=15) response = nil begin # set user agent headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36" attempts=0 max_attempts=10 found = false uri = URI.parse uri_string unless uri _log error "Unable to parse URI from: #{uri_string}" return end until( found || attempts >= max_attempts) attempts+=1 #proxy_addr = "127.0.0.1" proxy_addr = nil #proxy_port = "8080" proxy_port = nil # set options opts = {} if uri.instance_of? URI::HTTPS opts[:use_ssl] = true opts[:verify_mode] = OpenSSL::SSL::VERIFY_NONE end http = Net::HTTP.start(uri.host, uri.port, proxy_addr, proxy_port, opts) #http.set_debug_output($stdout) http.read_timeout = 20 http.open_timeout = 20 path = "#{uri.path}" path = "/" if path=="" # add in the query parameters if uri.query path += "?#{uri.query}" end ### ALLOW DIFFERENT VERBS HERE if method == :get request = Net::HTTP::Get.new(uri) elsif method == :post # see: https://coderwall.com/p/c-mu-a/http-posts-in-ruby request = Net::HTTP::Post.new(uri) request.body = data elsif method == :head request = Net::HTTP::Head.new(uri) elsif method == :propfind request = Net::HTTP::Propfind.new(uri.request_uri) request.body = "Here's the body." # Set your body (data) request["Depth"] = "1" # Set your headers: one header per line. elsif method == :options request = Net::HTTP::Options.new(uri.request_uri) elsif method == :trace request = Net::HTTP::Trace.new(uri.request_uri) request.body = "intrigue" end ### END VERBS # set the headers headers.each do |k,v| request[k] = v end # handle credentials #if credentials # request.basic_auth(credentials[:username],credentials[:password]) #end # get the response response = http.request(request) if response.code=="200" break end if (response.header['location']!=nil) newuri=URI.parse(response.header['location']) if(newuri.relative?) newuri=uri+response.header['location'] end uri=newuri else found=true #resp was 404, etc end #end if location end #until ### TODO - this code may be be called outside the context of a task, ### meaning @task_result is not available to it. Below, we check to ### make sure that it exists before attempting to log anything, ### but there may be a cleaner way to do this (hopefully?). Maybe a ### global logger or logging queue? ### #rescue TypeError # # https://github.com/jaimeiniesta/metainspector/issues/125 # puts "TypeError - unknown failure" rescue ArgumentError => e puts "Unable to open connection: #{e}" rescue Net::OpenTimeout => e puts "Timeout : #{e}" rescue Net::ReadTimeout => e puts "Timeout : #{e}" rescue Errno::ETIMEDOUT => e puts "Timeout : #{e}" rescue Errno::EINVAL => e puts "Unable to connect: #{e}" rescue Errno::ENETUNREACH => e puts "Unable to connect: #{e}" rescue Errno::EHOSTUNREACH => e puts "Unable to connect: #{e}" rescue URI::InvalidURIError => e # # XXX - This is an issue. We should catch this and ensure it's not # due to an underscore / other acceptable character in the URI # http://stackoverflow.com/questions/5208851/is-there-a-workaround-to-open-urls-containing-underscores-in-ruby # puts "Unable to request URI: #{uri} #{e}" rescue OpenSSL::SSL::SSLError => e puts "SSL connect error : #{e}" rescue Errno::ECONNREFUSED => e puts "Unable to connect: #{e}" rescue Errno::ECONNRESET => e puts "Unable to connect: #{e}" rescue Net::HTTPBadResponse => e puts "Unable to connect: #{e}" rescue Zlib::BufError => e puts "Unable to connect: #{e}" rescue Zlib::DataError => e # "incorrect header check - may be specific to ruby 2.0" puts "Unable to connect: #{e}" rescue EOFError => e puts "Unable to connect: #{e}" rescue SocketError => e puts "Unable to connect: #{e}" #rescue SystemCallError => e # puts "Unable to connect: #{e}" #rescue ArgumentError => e # puts "Argument Error: #{e}" rescue Encoding::InvalidByteSequenceError => e puts "Encoding error: #{e}" rescue Encoding::UndefinedConversionError => e puts "Encoding error: #{e}" end response end
_match_http_response(check, response,options)
click to toggle source
this method takes a check and a net/http response object and constructs it into a format that's matchable. it then attempts to match, and returns a match object if it matches, otherwise returns nil.
# File lib/intrigue-ident.rb, line 181 def _match_http_response(check, response,options) # Construct an Intrigue Entity of type Uri so we can match it data = [] =begin json = '{ "id": 1572, "type": "Intrigue::Entity::Uri", "name": "http://69.162.37.69:80", "deleted": false, "hidden": false, "detail_string": "Server: | App: | Title: Index page", "details": { "uri": "http://69.112.37.69:80", "code": "200", "port": 80, "forms": false, "title": "Index page", "generator": "Whatever", "verbs": null, "headers": ["content-length: 701", "last-modified: Tue, 03 Jul 2018 16:55:36 GMT", "cache-control: no-cache", "content-type: text/html"], "host_id": 1571, "scripts": [], "products": [], "cookies": "", "protocol": "tcp", "ip_address": "69.112.37.69", "javascript": [], "fingerprint": [], "api_endpoint": false, "masscan_string": "sudo masscan -p80,443,2004,3389,7001,8000,8080,8081,8443,U:161,U:500 --max-rate 10000 -oL /tmp/masscan20180703-9816-18n0ri --range 69.162.0.0/18", "app_fingerprint": [], "hidden_original": "http://69.162.37.69:80", "response_data_hash": "7o0r6ie5DOrJJnz1sS7RGO4XWsNn3hWykbwGkGnySWU=", "server_fingerprint": [], "enrichment_complete": ["enrich/uri"], "include_fingerprint": [], "enrichment_scheduled": ["enrich/uri"], "hidden_response_data": "", "hidden_screenshot_contents": """ }, "generated_at": "2018-07-04T03:43:11+00:00" }' =end data = {} data["details"] = {} data["details"]["hidden_response_data"] = "#{response.body}" # construct the headers into a big string block headers = [] response.each_header do |h,v| headers << "#{h}: #{v}" end data["details"]["headers"] = headers ### grab the page attributes match = response.body.match(/<title>(.*?)<\/title>/i) data["details"]["title"] = match.captures.first if match match = response.body.match(/<meta name="generator" content=(.*?)>/i) data["details"]["generator"] = match.captures.first.gsub("\"","") if match data["details"]["cookies"] = response.header['set-cookie'] data["details"]["response_data_hash"] = Digest::SHA256.base64digest("#{response.body}") # call the actual matcher & return _match_uri_hash check, data, options end
_match_uri_hash(check, data, options={})
click to toggle source
# File lib/intrigue-ident.rb, line 134 def _match_uri_hash(check, data, options={}) return nil unless check && data # data[:body] => page body # data[:headers] => block of text with headers, one per line # data[:cookies] => set_cookie header # data[:title] => parsed page title # data[:generator] => parsed meta generator tag # data[:body_md5] => md5 hash of the body # if type "content", do the content check if check[:match_type] == :content_body if data["details"] && data["details"]["hidden_response_data"] match = _construct_match_response(check,data,options) if data["details"]["hidden_response_data"] =~ check[:match_content] end elsif check[:match_type] == :content_headers if data["details"] && data["details"]["headers"] match = _construct_match_response(check,data,options) if data["details"]["headers"].join("\n") =~ check[:match_content] end elsif check[:match_type] == :content_cookies # Check only the set-cookie header if data["details"] && data["details"]["cookies"] match = _construct_match_response(check,data,options) if data["details"]["cookies"] =~ check[:match_content] end elsif check[:match_type] == :content_generator # Check only the set-cookie header if data["details"] && data["details"]["generator"] match = _construct_match_response(check,data,options) if data["details"]["generator"] =~ check[:match_content] end elsif check[:match_type] == :content_title # Check only the set-cookie header if data["details"] && data["details"]["title"] match = _construct_match_response(check,data,options) if data["details"]["title"] =~ check[:match_content] end elsif check[:match_type] == :checksum_body if data["details"] && data["details"]["response_data_hash"] match = _construct_match_response(check,data,options) if Digest::MD5.hexdigest(data["details"]["response_data_hash"]) == check[:checksum] end end match end