module Wmap
Class to trace de-activated site. This is need for basic state tracking for our sites.
require “singleton”
Main utility module to provide the common functions across different classes
Utilities for wp_tracker class only; must use with other Utils
modules.
Constants
- GEM
- NAME
- VERSION
Attributes
Public Class Methods
URL checker - check the status of the remote URL
# File lib/wmap.rb, line 122 def check(url) checker=Wmap::UrlChecker.new(:verbose=>false) checker.url_worker(url) end
Crawler to search url contents for new sites
# File lib/wmap.rb, line 86 def crawl(url) crawler=Wmap::UrlCrawler.new crawler.crawl(url) end
# File lib/wmap.rb, line 71 def data_dir(data_path) @data_dir=data_path.to_s end
DNS Brute Forcer
# File lib/wmap.rb, line 186 def dns_brute(domain) bruter=Wmap::DnsBruter.new bruter.query(domain) end
Domain Tracking - check with the trust domain seed file locally, to determine if it's a new internet domain NOT to confuse with the Internet 'whois' lookup
# File lib/wmap.rb, line 140 def domain_known?(domain) tracker=Wmap::DomainTracker.instance if @data_dir tracker.data_dir=@data_dir tracker.domains_file=tracker.data_dir + "/" + "domains" tracker.load_domains_from_file(tracker.domains_file) end tracker.domain_known?(domain) end
Retrieve root domain from a host
# File lib/wmap.rb, line 192 def domain_root(host) Wmap::Utils.get_domain_root(host) end
Dump out the unique sites into a plain file
# File lib/wmap.rb, line 224 def dump(file) store=Wmap::SiteTracker.instance if @data_dir store.data_dir = @data_dir store.sites_file = searcher.data_dir + "/" + "sites" store.load_site_stores_from_file(searcher.sites_file) end store.save_uniq_sites(file) end
Dump out the unique sites into a XML file
# File lib/wmap.rb, line 235 def dump_xml(file) store=Wmap::SiteTracker.instance if @data_dir store.data_dir = @data_dir store.sites_file = searcher.data_dir + "/" + "sites" store.load_site_stores_from_file(searcher.sites_file) end store.save_uniq_sites_xml(file) end
GeoIP Tracking - check the host / IP against the GeoIP data repository, return the Geographic information if found
# File lib/wmap.rb, line 116 def geoip(host) tracker=Wmap::GeoIPTracker.new tracker.query(host) end
Search the Google engines and sort out sites known by Google
# File lib/wmap.rb, line 270 def google sites=Wmap::GoogleSearchScraper.new.workers.keys end
Host Tracking - check local hosts file to see if this is a hostname known from the host seed file NOT to confuse with a regular DNS lookup over the internet
# File lib/wmap.rb, line 152 def host_known?(host) tracker=Wmap::HostTracker.instance if @data_dir tracker.data_dir = data_dir tracker.hosts_file = tracker.data_dir + "/" + "hosts" tracker.load_known_hosts_from_file(tracker.hosts_file) end tracker.host_known?(host) end
IP Tracking - check local hosts file to see if this is an IP known from the seed file NOT to confuse with a regular reverse DNS lookup over the internet
# File lib/wmap.rb, line 175 def ip_known?(ip) tracker=Wmap::HostTracker.instance if @data_dir tracker.data_dir = data_dir tracker.hosts_file = tracker.data_dir + "/" + "hosts" tracker.load_known_hosts_from_file(tracker.hosts_file) end tracker.ip_known?(ip) end
Check if the IP is within the range of the known CIDR blocks
# File lib/wmap.rb, line 128 def ip_trusted?(ip) tracker=Wmap::CidrTracker.new if @data_dir tracker.data_dir=@data_dir tracker.cidr_seeds=tracker.data_dir + "/" + "cidrs" tracker.load_cidr_blks_from_file(tracker.cidr_seeds) end tracker.ip_trusted?(ip) end
Host-name mutation for catch easily guessable hostname, i.e. “ww1.example.com” => [“ww1,example.com”,“ww2.example.com”,…]
# File lib/wmap.rb, line 202 def mutation (host) Wmap::DnsBruter.new.hostname_mutation(host) end
Print a site's full information from the repository
# File lib/wmap.rb, line 275 def print(site) searcher=Wmap::SiteTracker.instance searcher.print_site(site) end
Print a site's full information from the repository
# File lib/wmap.rb, line 281 def print_all searcher=Wmap::SiteTracker.instance if @data_dir searcher.data_dir = @data_dir searcher.sites_file = searcher.data_dir + "/" + "sites" searcher.load_site_stores_from_file(searcher.sites_file) end searcher.print_all_sites end
Simple parser for the project version file
# File lib/wmap.rb, line 42 def read_ver ver=Hash.new f=File.open(VERSION,'r') f.each do |line| line.chomp! case line when /^(\s)*#/ next when /\=/ entry=line.split("=").map! {|x| x.strip} ver[entry[0]]=entry[1] end end f.close return ver end
Refresh the site information in the local data repository
# File lib/wmap.rb, line 246 def refresh(site) store=Wmap::SiteTracker.instance if @data_dir store.data_dir = @data_dir store.sites_file = searcher.data_dir + "/" + "sites" store.load_site_stores_from_file(searcher.sites_file) end store.refresh(site) store.save! end
Refresh the site information in the local data repository
# File lib/wmap.rb, line 258 def refresh_all store=Wmap::SiteTracker.instance if @data_dir store.data_dir = @data_dir store.sites_file = searcher.data_dir + "/" + "sites" store.load_site_stores_from_file(searcher.sites_file) end store.refresh_all store.save! end
Check URL/Site response code
# File lib/wmap.rb, line 207 def response_code(url) checker=Wmap::UrlChecker.new checker.response_code(url) end
Fast tcp port scanner on a single host or IP
# File lib/wmap.rb, line 98 def scan(host) scanner=Wmap::PortScanner.new scanner.scan(host) end
Fast multi-processes tcp port scanner on a list of targets
# File lib/wmap.rb, line 104 def scans(target_list) scanner=Wmap::PortScanner.new scanner.scans(target_list) end
Search the site repository for all entries that match the pattern
# File lib/wmap.rb, line 213 def search(pattern) searcher=Wmap::SiteTracker.instance if @data_dir searcher.data_dir = @data_dir searcher.sites_file = searcher.data_dir + "/" + "sites" searcher.load_site_stores_from_file(searcher.sites_file) end searcher.search(pattern) end
Sub-domain tracking - check local hosts file to see if the sub-domain is already known
# File lib/wmap.rb, line 163 def sub_domain_known?(host) tracker=Wmap::HostTracker.instance if @data_dir tracker.data_dir = data_dir tracker.hosts_file = tracker.data_dir + "/" + "hosts" tracker.load_known_hosts_from_file(tracker.hosts_file) end tracker.sub_domain_known?(host) end
CIDR Tracking - check the host against the local CIDR seed file, return the CIDR tracking path if found
# File lib/wmap.rb, line 110 def track(host) tracker=Wmap::CidrTracker.new tracker.cidr_worker(host) end
whois query and sort the result into structured data
# File lib/wmap.rb, line 92 def whois(domain) whois=Wmap::Whois.new(:verbose=>false) whois.query(domain) end
Log the information into file
# File lib/wmap.rb, line 197 def wlog(msg,agent,log_file) Wmap::Utils.wlog(msg,agent,log_file) end
Explorer to discover and inventory web application / service automatically
# File lib/wmap.rb, line 76 def wmap(seed) if @data_dir cmd = "bin/wmap" + " -d " + @data_dir + " -t " + seed else cmd="bin/wmap" + " -t " + seed end system(cmd) end