class NexposeTicketing::TicketService

WARNING! This code is still rough and going through substantive changes. While

you can build tools using this library today, keep in mind that
method names and parameters may change in the future.

Constants

LOGGER_FILE
TICKET_SERVICE_CONFIG_PATH

Attributes

first_time[RW]
helper_data[RW]
nexpose_data[RW]
options[RW]
ticket_repository[RW]

Public Instance Methods

all_site_report(ticket_repository, options) click to toggle source

Generates a full site(s) report ticket(s).

# File lib/nexpose_ticketing/ticket_service.rb, line 153
def all_site_report(ticket_repository, options)
  group = "#{options[:scan_mode]}s"

  log_message("Generating full vulnerability report on user entered #{group}.")
  items_to_query = Array(options[group.to_sym])
  log_message("Generating full vulnerability report on the following #{group}: #{items_to_query.join(', ')}")
  
  items_to_query.each do |item|
    log_message("Running full vulnerability report on item #{item}")
    initial_scan_file = ticket_repository.generate_initial_scan_data(options,
                                                                     item)

    log_message('Preparing tickets.')
    nexpose_id = format_id(item)
    ticket_rate_limiter(initial_scan_file, 'create', nexpose_id)
    post_scan(item_id: item, generate_asset_list: true)
  end

  log_message('Finished processing all vulnerabilities.')
end
delta_new_scan(item_id, options, scan_histories) click to toggle source

Performs a delta scan

# File lib/nexpose_ticketing/ticket_service.rb, line 361
def delta_new_scan(item_id, options, scan_histories)
  delta_func = "delta_#{options[:scan_mode]}_new_scan"
  self.send(delta_func, item_id, options, scan_histories)
end
delta_scan(scan_histories) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 343
def delta_scan(scan_histories)
  log_message('Obtaining last scan information.')
  @latest_scans = @ticket_repository.last_scans(@options)

  # Scan states can change during our processing. Store the state we are
  # about to process and move this to the historical file if we
  # successfully process.
  log_message('Calculated deltas, storing current scan state.')
  @current_scan_state = ticket_repository.load_last_scans(@options)

  # Only run if a scan has been ran ever in Nexpose.
  return if @latest_scans.empty?

  delta_site_report(@ticket_repository, @options, scan_histories)
  log_message('Historical CSV file updated.')
end
delta_site_new_scan(nexpose_item, options, file_site_histories, tag_id=nil) click to toggle source

There's a new scan with possibly new vulnerabilities.

# File lib/nexpose_ticketing/ticket_service.rb, line 367
def delta_site_new_scan(nexpose_item, options, file_site_histories, tag_id=nil)
  log_message("New scan detected for nexpose id: #{nexpose_item}. Generating report.")
  
  format_method = "format_#{options[:scan_mode]}_id"
  nexpose_id = self.send(format_method, tag_id || nexpose_item)

  log_message("Scan id for new scan: #{file_site_histories[nexpose_item]}.")

  if @mode.updates_supported?
    helper_method = 'update'
    old_vulns_mode = 'old_ticket'
  else
    helper_method = 'create'
    old_vulns_mode = 'old'
  end

  scan_options = { scan_id: file_site_histories[nexpose_item],
                   nexpose_item: nexpose_item,
                   severity: options[:severity],
                   ticket_mode: options[:ticket_mode],
                   riskScore: options[:riskScore],
                   vulnerabilityCategories: options[:vulnerabilityCategories],
                   tag_run: options[:tag_run],
                   tag: tag_id,
                   old_vulns_mode: old_vulns_mode }

  close_tickets = (options[:close_old_tickets_on_update] == 'Y')

  csv_files = @ticket_repository.generate_delta_csv(scan_options,
                                                    close_tickets)
  delta_scan_file = csv_files[:new_csv]

  log_message('Preparing tickets.')

  ticket_rate_limiter(delta_scan_file, helper_method, nexpose_id)
    
  return unless close_tickets

  old_vulns_file = csv_files[:old_csv]

  ticket_rate_limiter(old_vulns_file, 'close', nexpose_id)
end
delta_site_report(ticket_repository, options, scan_histories) click to toggle source

There's possibly a new scan with new data.

# File lib/nexpose_ticketing/ticket_service.rb, line 175
def delta_site_report(ticket_repository, options, scan_histories)
  # Compares the scan information from file && Nexpose.
  no_processing = true
  @latest_scans.each do |item_id, last_scan_id|
    prev_scan_id = scan_histories[item_id]

    # There's no entry in the file, so it's either a new item in Nexpose or a new item we have to monitor.
    if prev_scan_id.nil? || prev_scan_id == -1
      options[:nexpose_item] = item_id
      full_new_site_report(item_id, ticket_repository, options)
      options[:nexpose_item] = nil
      post_scan(item_id: item_id, generate_asset_list: true)
      no_processing = false
    # Site has been scanned since last seen according to the file.
    elsif prev_scan_id.to_s != @latest_scans[item_id].to_s
      delta_new_scan(item_id, options, scan_histories)
      post_scan item_id: item_id
      no_processing = false
    end
  end

  log_name = @options["#{@options[:scan_mode]}_file_name".to_sym]
  # Done processing, update the CSV to the latest scan info.
  if no_processing
    log_message("Nothing new to process, historical CSV file has not been updated: #{options[:file_name]}.") 
  else
    log_message("Done processing, historical CSV file has been updated: #{options[:file_name]}.")
  end
  no_processing
end
delta_tag_new_scan(nexpose_item, options, file_site_histories, tag_id=nil) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 410
def delta_tag_new_scan(nexpose_item, options, file_site_histories, tag_id=nil)
  # It's a tag run and something has changed (new/removed asset or new scan ID for an asset). To find out what, we must compare
  # All tag assets and their scan IDs. Firstly we fetch all the assets in the tags
  # in the configuration file and store them temporarily
  item_id = nexpose_item
  tag_assets_tmp_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.tmp")
  tag_assets_historic_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.csv")
  ticket_repository.generate_tag_asset_list(tags: item_id,
                          csv_file: tag_assets_tmp_file)
  new_tag_configuration = ticket_repository.read_tag_asset_list(tag_assets_tmp_file)
  historic_tag_config = ticket_repository.read_tag_asset_list(tag_assets_historic_file)
  #Compare the assets within the tags and their scan histories to find the ones we need to query
  changed_assets = Hash[*(historic_tag_config.to_a - new_tag_configuration.to_a).flatten]
  new_assets = Hash[*(new_tag_configuration.to_a - historic_tag_config.to_a).flatten]
  new_assets.delete_if {|asset_id, scan_id| historic_tag_config.has_key?(asset_id.to_s)}

  #all_assets_changed = new_assets.merge(changed_assets)
  changed_assets.each do |asset_id, scan_id|
    delta_site_new_scan(asset_id, options, changed_assets, item_id)
  end

  new_assets.each do |asset_id, scan_id|
    #Since no previous scan IDs - we generate a full report.
    options[:nexpose_item] = asset_id
    full_new_site_report(item_id, ticket_repository, options)
    options.delete(:nexpose_item)
  end

   #Update the historic file
  new_tag_asset_list = historic_tag_config.merge(new_tag_configuration)
  trimmed_csv = []
  trimmed_csv << 'asset_id, last_scan_id'
  new_tag_asset_list.each do |asset_id, last_scan_id|
    trimmed_csv << "#{asset_id},#{last_scan_id}"
  end
  ticket_repository.save_to_file(tag_assets_historic_file, trimmed_csv)
  File.delete(tag_assets_tmp_file)
end
format_id(item_id) click to toggle source

Formats the Nexpose item ID according to the asset grouping mode

# File lib/nexpose_ticketing/ticket_service.rb, line 506
def format_id(item_id)
  self.send("format_#{options[:scan_mode]}_id", item_id)
end
format_site_id(item_id) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 510
def format_site_id(item_id)
  item_id
end
format_tag_id(item_id) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 514
def format_tag_id(item_id)
  "T#{item_id}"
end
full_new_site_report(nexpose_item, ticket_repository, options) click to toggle source

There's a new site we haven't seen before.

# File lib/nexpose_ticketing/ticket_service.rb, line 207
def full_new_site_report(nexpose_item, ticket_repository, options)
  log_message("New nexpose id: #{nexpose_item} detected. Generating report.")
  options[:scan_id] = 0

  initial_scan_file = ticket_repository.generate_initial_scan_data(options,
                                                                   nexpose_item)
  log_message('Preparing tickets.')

  nexpose_id = format_id(nexpose_item)
  ticket_rate_limiter(initial_scan_file, 'create', nexpose_id)
end
full_scan() click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 333
def full_scan
  log_message('Storing current scan state before obtaining all vulnerabilities.')
  @current_scan_state = ticket_repository.load_last_scans(@options)
  
  all_site_report(@ticket_repository, @options)

  #Generate historical CSV file after completing the fist query.
  log_message('Historical CSV file generated.')
end
full_scan_required?(histories) click to toggle source

Determines whether all assets must be scanned

# File lib/nexpose_ticketing/ticket_service.rb, line 519
def full_scan_required?(histories)
  self.send("full_#{@options[:scan_mode]}_scan_required?", histories)
end
full_site_scan_required?(scan_histories) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 523
def full_site_scan_required?(scan_histories)
  is_full_run = false

  if @options[:sites].nil? || @options[:sites].empty?
    is_full_run = true
    
    all_site_details = @ticket_repository.all_site_details
    @options[:sites] = all_site_details.map { |s| s.id.to_s }
   
    log_message("List of sites is now <#{@options[:sites]}>")
  end

  is_full_run || scan_histories.nil?
end
full_tag_scan_required?(scan_histories) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 538
def full_tag_scan_required?(scan_histories)
  if @options[:tags].nil? || @options[:tags].empty?
    fail 'No tags specified within the configuration.'  
  end
  return scan_histories.nil?
end
get_scan_mode() click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 295
def get_scan_mode
  return 'tag' unless @options[:tags].nil? || @options[:tags].empty?
  return 'site'
end
get_site_file_header() click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 497
def get_site_file_header
  ['site_id,last_scan_id,finished']
end
get_solutions() click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 319
def get_solutions
  log_message('Retrieving solutions from Nexpose')
  store = Store.new
  return store if Store.store_exists?

  store.set_path(@ticket_repository.get_solution_data)

  log_message('Parsing and storing solutions.')
  store.fill_store

  log_message('Solution store created.')
  store
end
get_tag_file_header() click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 501
def get_tag_file_header
  ['tag_id,last_scan_fingerprint']
end
load_class(type, name) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 102
def load_class(type, name)
  name.gsub!(type.capitalize, '')
  path = "#{type}s/#{name}_#{type}.rb".downcase
  
  log_message("Loading #{type} dependency: #{path}.")
  begin 
    require_relative path
  rescue => e
    error = "#{type.capitalize} dependency '#{path}' could not be loaded."
    @log.error e.to_s
    @log.error error
    fail error
  end

  eval("#{name}#{type.capitalize}")
end
log_message(message) click to toggle source

Logs a message if logging is enabled.

# File lib/nexpose_ticketing/ticket_service.rb, line 135
def log_message(message)
  @log.info(message) if @options[:logging_enabled]
end
post_scan(**modifiers) click to toggle source

Methods to run after a scan

# File lib/nexpose_ticketing/ticket_service.rb, line 450
def post_scan(**modifiers)
   self.send("post_#{@options[:scan_mode]}_scan", modifiers)
   scan_history = self.send("get_#{@options[:scan_mode]}_file_header")

   item_id = modifiers[:item_id]
   historic_data = nil
   if File.exists?(@historical_file)
     log_message("Updating historical CSV file: #{@historical_file}.")
     historic_data = []
     CSV.foreach(@historical_file, headers: true) { |r| historic_data << r }
   end

   updated_row = [@current_scan_state.find { |row| row[0].eql?(item_id) }]

   if historic_data.nil?
     log_message('No historical CSV file found. Generating.')
     scan_history.concat(updated_row)
   else
     index = historic_data.find_index { |id| id[0] == item_id }
     if index.nil?
       historic_data.concat(updated_row)
       historic_data.sort! { |x,y| x[0].to_i <=> y[0].to_i }
     else
       historic_data[index] = updated_row
       historic_data.flatten!
     end
     scan_history.concat(historic_data)
   end
   
   log_message('Updated historical CSV file for ' \
               "#{@options[:scan_mode]}: #{item_id}.")
   @ticket_repository.save_to_file(@historical_file, scan_history)
 end
post_site_scan(**modifiers) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 484
def post_site_scan(**modifiers)
end
post_tag_scan(**modifiers) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 487
def post_tag_scan(**modifiers)
  return unless modifiers[:generate_asset_list]      
  file_name = "#{@options[:tag_file_name]}_#{modifiers[:item_id]}.csv"
  historic_file = File.join(File.dirname(__FILE__), 'tag_assets', file_name)

  log_message("Generating current tag asset file: #{historic_file}.")
  ticket_repository.generate_tag_asset_list(tags: modifiers[:item_id],
                                            csv_file: historic_file)
end
prepare_historical_data(ticket_repository, options) click to toggle source

Prepares all the local and nexpose historical data.

# File lib/nexpose_ticketing/ticket_service.rb, line 140
def prepare_historical_data(ticket_repository, options)
  historical_scan_file = @historical_file

  file_site_histories = nil
  if File.exists?(historical_scan_file)
    log_message("Reading historical CSV file: #{historical_scan_file}.")
    file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
  end

  file_site_histories
end
setup(helper_data) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 67
def setup(helper_data)
  service_data = ConfigParser.get_config(TICKET_SERVICE_CONFIG_PATH)

  @helper_data = helper_data
  @nexpose_data = service_data[:nexpose_data]
  @options = service_data[:options]
  @options[:file_name] = @options[:file_name].to_s
  @options[:scan_mode] = get_scan_mode
  
  #Temporary - this should be refactored out e.g. to include DAGs
  @options[:tag_run] = @options[:scan_mode] == 'tag'

  file_name = @options["#{@options[:scan_mode]}_file_name".to_sym]
  @historical_file = File.join(File.dirname(__FILE__), file_name)

  # Sets logging up, if enabled.
  setup_logging(@options[:logging_enabled])


  mode_class = load_class 'mode', @options[:ticket_mode]
  @mode = mode_class.new(@options)

  @options[:query_suffix] = @mode.get_query_suffix

  helper_class = load_class 'helper', @helper_data[:helper_name]
  @helper = helper_class.new(@helper_data, @options, @mode)

  log_message("Creating ticketing repository with timeout value: #{@options[:timeout]}.")
  @ticket_repository = NexposeTicketing::TicketRepository.new(options)
  @ticket_repository.nexpose_login(@nexpose_data)

  solutions = get_solutions
  @mode.set_solution_store solutions
end
setup_logging(enabled = false) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 119
def setup_logging(enabled = false)
  helper_log = NexposeTicketing::NxLogger.instance
  helper_log.setup_logging(@options[:logging_enabled],
                           @options[:log_level],
                           @options[:log_console])

  return unless enabled
  require 'logger'
  directory = File.dirname(LOGGER_FILE)
  FileUtils.mkdir_p(directory) unless File.directory?(directory)
  @log = Logger.new(LOGGER_FILE, 'monthly')
  @log.level = Logger::INFO
  log_message('Logging enabled, starting service.')
end
start() click to toggle source

Starts the Ticketing Service.

# File lib/nexpose_ticketing/ticket_service.rb, line 301
def start
  # Checks if the csv historical file already exists and reads it, otherwise create it and assume first time run.
  scan_histories = prepare_historical_data(@ticket_repository, @options)


  # If we didn't specify a site || first time run (no scan history), then it gets all the vulnerabilities.
  @options[:initial_run] = full_scan_required?(scan_histories)

  if @options[:initial_run]
    full_scan
  else
    delta_scan(scan_histories)
  end

  @helper.finish
  log_message('Exiting ticket service.')
end
ticket_rate_limiter(query_results_file, ticket_method, nexpose_item) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 241
def ticket_rate_limiter(query_results_file, ticket_method, nexpose_item)
  batch_size_max = @options[:batch_size]
  max_tickets = @options[:batch_ticket_limit]

  log_message("Batching tickets in sizes: #{@options[:batch_size]}")
  fields = @mode.get_matching_fields
  current_ids = Hash[*fields.collect { |k| [k, nil] }.flatten]

  # Start the batching
  query_results_file.rewind
  csv_header = query_results_file.readline
  batch = []
  individual_ticket = []
  ticket_count = 0
  prev_row = nil

  begin
    CSV.foreach(query_results_file, headers: csv_header) do |row|
      if prev_row.nil?
        # First row of a ticket
        prev_row = row
        ticket_count = ticket_count + 1
        individual_ticket = [row]
      elsif fields.any? { |k| prev_row[k].nil? || prev_row[k] != row[k] }
        # New ticket found
        prev_row = nil
        batch.concat individual_ticket

        if batch.count >= batch_size_max || ticket_count >= max_tickets
          ticket_rate_limiter_processor(batch, ticket_method, nexpose_item)

          ticket_count = 0
          batch.clear
          batch << csv_header
        end

        redo
      else
        # Another row for existing ticket
        individual_ticket << row
      end
    end
  ensure
    log_message('Finished reading report. Sending any remaining tickets and cleaning up file system.')

    # Finish adding to the batch
    batch.concat individual_ticket
    ticket_rate_limiter_processor(batch, ticket_method, nexpose_item)

    query_results_file.close
    query_results_file.unlink
  end 
end
ticket_rate_limiter_processor(ticket_batch, ticket_method, nexpose_item) click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 219
def ticket_rate_limiter_processor(ticket_batch, ticket_method, nexpose_item)
  #Just the header (no tickets).
  if ticket_batch.size == 1
    log_message('Received empty batch. Not sending tickets.')
    return
  end

  nexpose_item = format_id(nexpose_item)

  # Prep the batch of tickets
  log_message("Preparing to #{ticket_method} tickets.")
  tickets = @helper.send("prepare_#{ticket_method}_tickets", 
                         ticket_batch.join(''), 
                         nexpose_item)
  log_message("Parsed rows: #{ticket_batch.size}")

  # Send them off
  log_message('Sending tickets.')
  @helper.send("#{ticket_method}_tickets", tickets)
  log_message('Returning for next batch.')
end