class NexposeTicketing::TicketService
WARNING! This code is still rough and going through substantive changes. While
you can build tools using this library today, keep in mind that method names and parameters may change in the future.
Constants
- LOGGER_FILE
- TICKET_SERVICE_CONFIG_PATH
Attributes
first_time[RW]
helper_data[RW]
nexpose_data[RW]
options[RW]
ticket_repository[RW]
Public Instance Methods
all_site_report(ticket_repository, options)
click to toggle source
Generates a full site(s) report ticket(s).
# File lib/nexpose_ticketing/ticket_service.rb, line 153 def all_site_report(ticket_repository, options) group = "#{options[:scan_mode]}s" log_message("Generating full vulnerability report on user entered #{group}.") items_to_query = Array(options[group.to_sym]) log_message("Generating full vulnerability report on the following #{group}: #{items_to_query.join(', ')}") items_to_query.each do |item| log_message("Running full vulnerability report on item #{item}") initial_scan_file = ticket_repository.generate_initial_scan_data(options, item) log_message('Preparing tickets.') nexpose_id = format_id(item) ticket_rate_limiter(initial_scan_file, 'create', nexpose_id) post_scan(item_id: item, generate_asset_list: true) end log_message('Finished processing all vulnerabilities.') end
delta_new_scan(item_id, options, scan_histories)
click to toggle source
Performs a delta scan
# File lib/nexpose_ticketing/ticket_service.rb, line 361 def delta_new_scan(item_id, options, scan_histories) delta_func = "delta_#{options[:scan_mode]}_new_scan" self.send(delta_func, item_id, options, scan_histories) end
delta_scan(scan_histories)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 343 def delta_scan(scan_histories) log_message('Obtaining last scan information.') @latest_scans = @ticket_repository.last_scans(@options) # Scan states can change during our processing. Store the state we are # about to process and move this to the historical file if we # successfully process. log_message('Calculated deltas, storing current scan state.') @current_scan_state = ticket_repository.load_last_scans(@options) # Only run if a scan has been ran ever in Nexpose. return if @latest_scans.empty? delta_site_report(@ticket_repository, @options, scan_histories) log_message('Historical CSV file updated.') end
delta_site_new_scan(nexpose_item, options, file_site_histories, tag_id=nil)
click to toggle source
There's a new scan with possibly new vulnerabilities.
# File lib/nexpose_ticketing/ticket_service.rb, line 367 def delta_site_new_scan(nexpose_item, options, file_site_histories, tag_id=nil) log_message("New scan detected for nexpose id: #{nexpose_item}. Generating report.") format_method = "format_#{options[:scan_mode]}_id" nexpose_id = self.send(format_method, tag_id || nexpose_item) log_message("Scan id for new scan: #{file_site_histories[nexpose_item]}.") if @mode.updates_supported? helper_method = 'update' old_vulns_mode = 'old_ticket' else helper_method = 'create' old_vulns_mode = 'old' end scan_options = { scan_id: file_site_histories[nexpose_item], nexpose_item: nexpose_item, severity: options[:severity], ticket_mode: options[:ticket_mode], riskScore: options[:riskScore], vulnerabilityCategories: options[:vulnerabilityCategories], tag_run: options[:tag_run], tag: tag_id, old_vulns_mode: old_vulns_mode } close_tickets = (options[:close_old_tickets_on_update] == 'Y') csv_files = @ticket_repository.generate_delta_csv(scan_options, close_tickets) delta_scan_file = csv_files[:new_csv] log_message('Preparing tickets.') ticket_rate_limiter(delta_scan_file, helper_method, nexpose_id) return unless close_tickets old_vulns_file = csv_files[:old_csv] ticket_rate_limiter(old_vulns_file, 'close', nexpose_id) end
delta_site_report(ticket_repository, options, scan_histories)
click to toggle source
There's possibly a new scan with new data.
# File lib/nexpose_ticketing/ticket_service.rb, line 175 def delta_site_report(ticket_repository, options, scan_histories) # Compares the scan information from file && Nexpose. no_processing = true @latest_scans.each do |item_id, last_scan_id| prev_scan_id = scan_histories[item_id] # There's no entry in the file, so it's either a new item in Nexpose or a new item we have to monitor. if prev_scan_id.nil? || prev_scan_id == -1 options[:nexpose_item] = item_id full_new_site_report(item_id, ticket_repository, options) options[:nexpose_item] = nil post_scan(item_id: item_id, generate_asset_list: true) no_processing = false # Site has been scanned since last seen according to the file. elsif prev_scan_id.to_s != @latest_scans[item_id].to_s delta_new_scan(item_id, options, scan_histories) post_scan item_id: item_id no_processing = false end end log_name = @options["#{@options[:scan_mode]}_file_name".to_sym] # Done processing, update the CSV to the latest scan info. if no_processing log_message("Nothing new to process, historical CSV file has not been updated: #{options[:file_name]}.") else log_message("Done processing, historical CSV file has been updated: #{options[:file_name]}.") end no_processing end
delta_tag_new_scan(nexpose_item, options, file_site_histories, tag_id=nil)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 410 def delta_tag_new_scan(nexpose_item, options, file_site_histories, tag_id=nil) # It's a tag run and something has changed (new/removed asset or new scan ID for an asset). To find out what, we must compare # All tag assets and their scan IDs. Firstly we fetch all the assets in the tags # in the configuration file and store them temporarily item_id = nexpose_item tag_assets_tmp_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.tmp") tag_assets_historic_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.csv") ticket_repository.generate_tag_asset_list(tags: item_id, csv_file: tag_assets_tmp_file) new_tag_configuration = ticket_repository.read_tag_asset_list(tag_assets_tmp_file) historic_tag_config = ticket_repository.read_tag_asset_list(tag_assets_historic_file) #Compare the assets within the tags and their scan histories to find the ones we need to query changed_assets = Hash[*(historic_tag_config.to_a - new_tag_configuration.to_a).flatten] new_assets = Hash[*(new_tag_configuration.to_a - historic_tag_config.to_a).flatten] new_assets.delete_if {|asset_id, scan_id| historic_tag_config.has_key?(asset_id.to_s)} #all_assets_changed = new_assets.merge(changed_assets) changed_assets.each do |asset_id, scan_id| delta_site_new_scan(asset_id, options, changed_assets, item_id) end new_assets.each do |asset_id, scan_id| #Since no previous scan IDs - we generate a full report. options[:nexpose_item] = asset_id full_new_site_report(item_id, ticket_repository, options) options.delete(:nexpose_item) end #Update the historic file new_tag_asset_list = historic_tag_config.merge(new_tag_configuration) trimmed_csv = [] trimmed_csv << 'asset_id, last_scan_id' new_tag_asset_list.each do |asset_id, last_scan_id| trimmed_csv << "#{asset_id},#{last_scan_id}" end ticket_repository.save_to_file(tag_assets_historic_file, trimmed_csv) File.delete(tag_assets_tmp_file) end
format_id(item_id)
click to toggle source
Formats the Nexpose item ID according to the asset grouping mode
# File lib/nexpose_ticketing/ticket_service.rb, line 506 def format_id(item_id) self.send("format_#{options[:scan_mode]}_id", item_id) end
format_site_id(item_id)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 510 def format_site_id(item_id) item_id end
format_tag_id(item_id)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 514 def format_tag_id(item_id) "T#{item_id}" end
full_new_site_report(nexpose_item, ticket_repository, options)
click to toggle source
There's a new site we haven't seen before.
# File lib/nexpose_ticketing/ticket_service.rb, line 207 def full_new_site_report(nexpose_item, ticket_repository, options) log_message("New nexpose id: #{nexpose_item} detected. Generating report.") options[:scan_id] = 0 initial_scan_file = ticket_repository.generate_initial_scan_data(options, nexpose_item) log_message('Preparing tickets.') nexpose_id = format_id(nexpose_item) ticket_rate_limiter(initial_scan_file, 'create', nexpose_id) end
full_scan()
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 333 def full_scan log_message('Storing current scan state before obtaining all vulnerabilities.') @current_scan_state = ticket_repository.load_last_scans(@options) all_site_report(@ticket_repository, @options) #Generate historical CSV file after completing the fist query. log_message('Historical CSV file generated.') end
full_scan_required?(histories)
click to toggle source
Determines whether all assets must be scanned
# File lib/nexpose_ticketing/ticket_service.rb, line 519 def full_scan_required?(histories) self.send("full_#{@options[:scan_mode]}_scan_required?", histories) end
full_site_scan_required?(scan_histories)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 523 def full_site_scan_required?(scan_histories) is_full_run = false if @options[:sites].nil? || @options[:sites].empty? is_full_run = true all_site_details = @ticket_repository.all_site_details @options[:sites] = all_site_details.map { |s| s.id.to_s } log_message("List of sites is now <#{@options[:sites]}>") end is_full_run || scan_histories.nil? end
full_tag_scan_required?(scan_histories)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 538 def full_tag_scan_required?(scan_histories) if @options[:tags].nil? || @options[:tags].empty? fail 'No tags specified within the configuration.' end return scan_histories.nil? end
get_scan_mode()
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 295 def get_scan_mode return 'tag' unless @options[:tags].nil? || @options[:tags].empty? return 'site' end
get_site_file_header()
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 497 def get_site_file_header ['site_id,last_scan_id,finished'] end
get_solutions()
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 319 def get_solutions log_message('Retrieving solutions from Nexpose') store = Store.new return store if Store.store_exists? store.set_path(@ticket_repository.get_solution_data) log_message('Parsing and storing solutions.') store.fill_store log_message('Solution store created.') store end
get_tag_file_header()
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 501 def get_tag_file_header ['tag_id,last_scan_fingerprint'] end
load_class(type, name)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 102 def load_class(type, name) name.gsub!(type.capitalize, '') path = "#{type}s/#{name}_#{type}.rb".downcase log_message("Loading #{type} dependency: #{path}.") begin require_relative path rescue => e error = "#{type.capitalize} dependency '#{path}' could not be loaded." @log.error e.to_s @log.error error fail error end eval("#{name}#{type.capitalize}") end
log_message(message)
click to toggle source
Logs a message if logging is enabled.
# File lib/nexpose_ticketing/ticket_service.rb, line 135 def log_message(message) @log.info(message) if @options[:logging_enabled] end
post_scan(**modifiers)
click to toggle source
Methods to run after a scan
# File lib/nexpose_ticketing/ticket_service.rb, line 450 def post_scan(**modifiers) self.send("post_#{@options[:scan_mode]}_scan", modifiers) scan_history = self.send("get_#{@options[:scan_mode]}_file_header") item_id = modifiers[:item_id] historic_data = nil if File.exists?(@historical_file) log_message("Updating historical CSV file: #{@historical_file}.") historic_data = [] CSV.foreach(@historical_file, headers: true) { |r| historic_data << r } end updated_row = [@current_scan_state.find { |row| row[0].eql?(item_id) }] if historic_data.nil? log_message('No historical CSV file found. Generating.') scan_history.concat(updated_row) else index = historic_data.find_index { |id| id[0] == item_id } if index.nil? historic_data.concat(updated_row) historic_data.sort! { |x,y| x[0].to_i <=> y[0].to_i } else historic_data[index] = updated_row historic_data.flatten! end scan_history.concat(historic_data) end log_message('Updated historical CSV file for ' \ "#{@options[:scan_mode]}: #{item_id}.") @ticket_repository.save_to_file(@historical_file, scan_history) end
post_site_scan(**modifiers)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 484 def post_site_scan(**modifiers) end
post_tag_scan(**modifiers)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 487 def post_tag_scan(**modifiers) return unless modifiers[:generate_asset_list] file_name = "#{@options[:tag_file_name]}_#{modifiers[:item_id]}.csv" historic_file = File.join(File.dirname(__FILE__), 'tag_assets', file_name) log_message("Generating current tag asset file: #{historic_file}.") ticket_repository.generate_tag_asset_list(tags: modifiers[:item_id], csv_file: historic_file) end
prepare_historical_data(ticket_repository, options)
click to toggle source
Prepares all the local and nexpose historical data.
# File lib/nexpose_ticketing/ticket_service.rb, line 140 def prepare_historical_data(ticket_repository, options) historical_scan_file = @historical_file file_site_histories = nil if File.exists?(historical_scan_file) log_message("Reading historical CSV file: #{historical_scan_file}.") file_site_histories = ticket_repository.read_last_scans(historical_scan_file) end file_site_histories end
setup(helper_data)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 67 def setup(helper_data) service_data = ConfigParser.get_config(TICKET_SERVICE_CONFIG_PATH) @helper_data = helper_data @nexpose_data = service_data[:nexpose_data] @options = service_data[:options] @options[:file_name] = @options[:file_name].to_s @options[:scan_mode] = get_scan_mode #Temporary - this should be refactored out e.g. to include DAGs @options[:tag_run] = @options[:scan_mode] == 'tag' file_name = @options["#{@options[:scan_mode]}_file_name".to_sym] @historical_file = File.join(File.dirname(__FILE__), file_name) # Sets logging up, if enabled. setup_logging(@options[:logging_enabled]) mode_class = load_class 'mode', @options[:ticket_mode] @mode = mode_class.new(@options) @options[:query_suffix] = @mode.get_query_suffix helper_class = load_class 'helper', @helper_data[:helper_name] @helper = helper_class.new(@helper_data, @options, @mode) log_message("Creating ticketing repository with timeout value: #{@options[:timeout]}.") @ticket_repository = NexposeTicketing::TicketRepository.new(options) @ticket_repository.nexpose_login(@nexpose_data) solutions = get_solutions @mode.set_solution_store solutions end
setup_logging(enabled = false)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 119 def setup_logging(enabled = false) helper_log = NexposeTicketing::NxLogger.instance helper_log.setup_logging(@options[:logging_enabled], @options[:log_level], @options[:log_console]) return unless enabled require 'logger' directory = File.dirname(LOGGER_FILE) FileUtils.mkdir_p(directory) unless File.directory?(directory) @log = Logger.new(LOGGER_FILE, 'monthly') @log.level = Logger::INFO log_message('Logging enabled, starting service.') end
start()
click to toggle source
Starts the Ticketing Service.
# File lib/nexpose_ticketing/ticket_service.rb, line 301 def start # Checks if the csv historical file already exists and reads it, otherwise create it and assume first time run. scan_histories = prepare_historical_data(@ticket_repository, @options) # If we didn't specify a site || first time run (no scan history), then it gets all the vulnerabilities. @options[:initial_run] = full_scan_required?(scan_histories) if @options[:initial_run] full_scan else delta_scan(scan_histories) end @helper.finish log_message('Exiting ticket service.') end
ticket_rate_limiter(query_results_file, ticket_method, nexpose_item)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 241 def ticket_rate_limiter(query_results_file, ticket_method, nexpose_item) batch_size_max = @options[:batch_size] max_tickets = @options[:batch_ticket_limit] log_message("Batching tickets in sizes: #{@options[:batch_size]}") fields = @mode.get_matching_fields current_ids = Hash[*fields.collect { |k| [k, nil] }.flatten] # Start the batching query_results_file.rewind csv_header = query_results_file.readline batch = [] individual_ticket = [] ticket_count = 0 prev_row = nil begin CSV.foreach(query_results_file, headers: csv_header) do |row| if prev_row.nil? # First row of a ticket prev_row = row ticket_count = ticket_count + 1 individual_ticket = [row] elsif fields.any? { |k| prev_row[k].nil? || prev_row[k] != row[k] } # New ticket found prev_row = nil batch.concat individual_ticket if batch.count >= batch_size_max || ticket_count >= max_tickets ticket_rate_limiter_processor(batch, ticket_method, nexpose_item) ticket_count = 0 batch.clear batch << csv_header end redo else # Another row for existing ticket individual_ticket << row end end ensure log_message('Finished reading report. Sending any remaining tickets and cleaning up file system.') # Finish adding to the batch batch.concat individual_ticket ticket_rate_limiter_processor(batch, ticket_method, nexpose_item) query_results_file.close query_results_file.unlink end end
ticket_rate_limiter_processor(ticket_batch, ticket_method, nexpose_item)
click to toggle source
# File lib/nexpose_ticketing/ticket_service.rb, line 219 def ticket_rate_limiter_processor(ticket_batch, ticket_method, nexpose_item) #Just the header (no tickets). if ticket_batch.size == 1 log_message('Received empty batch. Not sending tickets.') return end nexpose_item = format_id(nexpose_item) # Prep the batch of tickets log_message("Preparing to #{ticket_method} tickets.") tickets = @helper.send("prepare_#{ticket_method}_tickets", ticket_batch.join(''), nexpose_item) log_message("Parsed rows: #{ticket_batch.size}") # Send them off log_message('Sending tickets.') @helper.send("#{ticket_method}_tickets", tickets) log_message('Returning for next batch.') end