module Wmap

Class to trace de-activated site. This is need for basic state tracking for our sites.

require “singleton”

Main utility module to provide the common functions across different classes

Utilities for wp_tracker class only; must use with other Utils modules.

Constants

GEM
NAME
VERSION

Attributes

known_internet_domains[RW]
verbose[W]

Public Class Methods

banner() click to toggle source

Project banner in ASCII Art 'soft' format, courtesy to patorjk.com/software/taag/

check(url) click to toggle source

URL checker - check the status of the remote URL

# File lib/wmap.rb, line 122
def check(url)
        checker=Wmap::UrlChecker.new(:verbose=>false)
        checker.url_worker(url)
end
crawl(url) click to toggle source

Crawler to search url contents for new sites

# File lib/wmap.rb, line 86
def crawl(url)
        crawler=Wmap::UrlCrawler.new
        crawler.crawl(url)
end
data_dir(data_path) click to toggle source
# File lib/wmap.rb, line 71
def data_dir(data_path)
  @data_dir=data_path.to_s
end
dns_brute(domain) click to toggle source

DNS Brute Forcer

# File lib/wmap.rb, line 186
def dns_brute(domain)
        bruter=Wmap::DnsBruter.new
        bruter.query(domain)
end
domain_known?(domain) click to toggle source

Domain Tracking - check with the trust domain seed file locally, to determine if it's a new internet domain NOT to confuse with the Internet 'whois' lookup

# File lib/wmap.rb, line 140
    def domain_known?(domain)
            tracker=Wmap::DomainTracker.instance
if @data_dir
  tracker.data_dir=@data_dir
  tracker.domains_file=tracker.data_dir + "/" + "domains"
  tracker.load_domains_from_file(tracker.domains_file)
end
            tracker.domain_known?(domain)
    end
domain_root(host) click to toggle source

Retrieve root domain from a host

# File lib/wmap.rb, line 192
def domain_root(host)
        Wmap::Utils.get_domain_root(host)
end
dump(file) click to toggle source

Dump out the unique sites into a plain file

# File lib/wmap.rb, line 224
  def dump(file)
                  store=Wmap::SiteTracker.instance
if @data_dir
  store.data_dir = @data_dir
  store.sites_file = searcher.data_dir + "/" + "sites"
  store.load_site_stores_from_file(searcher.sites_file)
end
                  store.save_uniq_sites(file)
  end
dump_xml(file) click to toggle source

Dump out the unique sites into a XML file

# File lib/wmap.rb, line 235
  def dump_xml(file)
                  store=Wmap::SiteTracker.instance
if @data_dir
  store.data_dir = @data_dir
  store.sites_file = searcher.data_dir + "/" + "sites"
  store.load_site_stores_from_file(searcher.sites_file)
end
store.save_uniq_sites_xml(file)
  end
geoip(host) click to toggle source

GeoIP Tracking - check the host / IP against the GeoIP data repository, return the Geographic information if found

# File lib/wmap.rb, line 116
def geoip(host)
        tracker=Wmap::GeoIPTracker.new
        tracker.query(host)
end
google() click to toggle source

Search the Google engines and sort out sites known by Google

# File lib/wmap.rb, line 270
def google
        sites=Wmap::GoogleSearchScraper.new.workers.keys
end
host_known?(host) click to toggle source

Host Tracking - check local hosts file to see if this is a hostname known from the host seed file NOT to confuse with a regular DNS lookup over the internet

# File lib/wmap.rb, line 152
    def host_known?(host)
            tracker=Wmap::HostTracker.instance
if @data_dir
  tracker.data_dir = data_dir
  tracker.hosts_file = tracker.data_dir + "/" + "hosts"
  tracker.load_known_hosts_from_file(tracker.hosts_file)
end
tracker.host_known?(host)
    end
ip_known?(ip) click to toggle source

IP Tracking - check local hosts file to see if this is an IP known from the seed file NOT to confuse with a regular reverse DNS lookup over the internet

# File lib/wmap.rb, line 175
    def ip_known?(ip)
            tracker=Wmap::HostTracker.instance
if @data_dir
  tracker.data_dir = data_dir
  tracker.hosts_file = tracker.data_dir + "/" + "hosts"
  tracker.load_known_hosts_from_file(tracker.hosts_file)
end
tracker.ip_known?(ip)
    end
ip_trusted?(ip) click to toggle source

Check if the IP is within the range of the known CIDR blocks

# File lib/wmap.rb, line 128
    def ip_trusted?(ip)
            tracker=Wmap::CidrTracker.new
if @data_dir
  tracker.data_dir=@data_dir
  tracker.cidr_seeds=tracker.data_dir + "/" + "cidrs"
  tracker.load_cidr_blks_from_file(tracker.cidr_seeds)
end
            tracker.ip_trusted?(ip)
    end
mutation(host) click to toggle source

Host-name mutation for catch easily guessable hostname, i.e. “ww1.example.com” => [“ww1,example.com”,“ww2.example.com”,…]

# File lib/wmap.rb, line 202
def mutation (host)
        Wmap::DnsBruter.new.hostname_mutation(host)
end
print(site) click to toggle source

Print a site's full information from the repository

print_all() click to toggle source

Print a site's full information from the repository

read_ver() click to toggle source

Simple parser for the project version file

# File lib/wmap.rb, line 42
def read_ver
        ver=Hash.new
        f=File.open(VERSION,'r')
        f.each do |line|
                line.chomp!
                case line
                when /^(\s)*#/
                        next
                when /\=/
                        entry=line.split("=").map! {|x| x.strip}
                        ver[entry[0]]=entry[1]
                end
        end
        f.close
        return ver
end
refresh(site) click to toggle source

Refresh the site information in the local data repository

# File lib/wmap.rb, line 246
  def refresh(site)
                  store=Wmap::SiteTracker.instance
if @data_dir
  store.data_dir = @data_dir
  store.sites_file = searcher.data_dir + "/" + "sites"
  store.load_site_stores_from_file(searcher.sites_file)
end
                  store.refresh(site)
                  store.save!
  end
refresh_all() click to toggle source

Refresh the site information in the local data repository

# File lib/wmap.rb, line 258
  def refresh_all
                  store=Wmap::SiteTracker.instance
if @data_dir
  store.data_dir = @data_dir
  store.sites_file = searcher.data_dir + "/" + "sites"
  store.load_site_stores_from_file(searcher.sites_file)
end
                  store.refresh_all
                  store.save!
  end
response_code(url) click to toggle source

Check URL/Site response code

# File lib/wmap.rb, line 207
def response_code(url)
        checker=Wmap::UrlChecker.new
        checker.response_code(url)
end
scan(host) click to toggle source

Fast tcp port scanner on a single host or IP

# File lib/wmap.rb, line 98
def scan(host)
        scanner=Wmap::PortScanner.new
        scanner.scan(host)
end
scans(target_list) click to toggle source

Fast multi-processes tcp port scanner on a list of targets

# File lib/wmap.rb, line 104
def scans(target_list)
        scanner=Wmap::PortScanner.new
        scanner.scans(target_list)
end
sub_domain_known?(host) click to toggle source

Sub-domain tracking - check local hosts file to see if the sub-domain is already known

# File lib/wmap.rb, line 163
    def sub_domain_known?(host)
            tracker=Wmap::HostTracker.instance
if @data_dir
  tracker.data_dir = data_dir
  tracker.hosts_file = tracker.data_dir + "/" + "hosts"
  tracker.load_known_hosts_from_file(tracker.hosts_file)
end
tracker.sub_domain_known?(host)
    end
track(host) click to toggle source

CIDR Tracking - check the host against the local CIDR seed file, return the CIDR tracking path if found

# File lib/wmap.rb, line 110
def track(host)
        tracker=Wmap::CidrTracker.new
        tracker.cidr_worker(host)
end
whois(domain) click to toggle source

whois query and sort the result into structured data

# File lib/wmap.rb, line 92
def whois(domain)
        whois=Wmap::Whois.new(:verbose=>false)
        whois.query(domain)
end
wlog(msg,agent,log_file) click to toggle source

Log the information into file

# File lib/wmap.rb, line 197
def wlog(msg,agent,log_file)
        Wmap::Utils.wlog(msg,agent,log_file)
end
wmap(seed) click to toggle source

Explorer to discover and inventory web application / service automatically

# File lib/wmap.rb, line 76
    def wmap(seed)
if @data_dir
  cmd = "bin/wmap" + " -d " + @data_dir + " -t " + seed
else
                cmd="bin/wmap" + " -t " + seed
end
            system(cmd)
    end