class Fluent::AnalyzeConfigFilter

Fluentd filter plugin to analyze configuration usage.

For documentation on inspecting parsed configuration elements, see www.rubydoc.info/github/fluent/fluentd/Fluent/Config/Element

Public Instance Methods

configure(conf) click to toggle source
Calls superclass method
# File lib/fluent/plugin/filter_analyze_config.rb, line 217
def configure(conf)
  super
  @log.info('analyze_config plugin: Starting to configure the plugin.')
  if File.file?(@google_fluentd_config_path) &&
     File.file?(@google_fluentd_baseline_config_path)
    @log.info(
      'analyze_config plugin: google-fluentd configuration file found at' \
      " #{@google_fluentd_config_path}. " \
      'google-fluentd baseline configuration file found at' \
      " #{@google_fluentd_baseline_config_path}. " \
      'google-fluentd Analyzing configuration.'
    )

    utils = Common::Utils.new(@log)
    platform = utils.detect_platform(true)
    project_id = utils.get_project_id(platform, nil)
    vm_id = utils.get_vm_id(platform, nil)
    zone = utils.get_location(platform, nil, true)

    # All metadata parameters must now be set.
    utils.check_required_metadata_variables(
      platform, project_id, zone, vm_id
    )

    # Retrieve monitored resource.
    # Fail over to retrieve monitored resource via the legacy path if we
    # fail to get it from Metadata Agent.
    resource = utils.determine_agent_level_monitored_resource_via_legacy(
      platform, nil, false, vm_id, zone
    )

    unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
      @monitoring_type
    )
      @log.warn(
        "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
        'unknown; there will be no metrics.'
      )
    end

    @registry = Monitoring::MonitoringRegistryFactory.create(
      @monitoring_type, project_id, resource, @gcm_service_address
    )
    # Export metrics every 60 seconds.
    timer_execute(:export_config_analysis_metrics, 60) do
      @registry.update_timestamps(PREFIX) if @registry.respond_to? :update_timestamps
      @registry.export
    end

    @log.info('analyze_config plugin: Registering counters.')
    enabled_plugins_counter = @registry.counter(
      :enabled_plugins,
      %i[plugin_name is_default_plugin has_default_config has_ruby_snippet],
      'Enabled plugins',
      PREFIX,
      'GAUGE'
    )
    @log.info(
      'analyze_config plugin: registered enable_plugins counter. ' \
      "#{enabled_plugins_counter}"
    )
    plugin_config_counter = @registry.counter(
      :plugin_config,
      %i[plugin_name param is_present has_default_config],
      'Configuration parameter usage for plugins relevant to Google Cloud.',
      PREFIX,
      'GAUGE'
    )
    @log.info('analyze_config plugin: registered plugin_config counter. ' \
      "#{plugin_config_counter}")
    config_bool_values_counter = @registry.counter(
      :config_bool_values,
      %i[plugin_name param value],
      'Values for bool parameters in Google Cloud plugins',
      PREFIX,
      'GAUGE'
    )
    @log.info('analyze_config plugin: registered config_bool_values ' \
      "counter. #{config_bool_values_counter}")

    config = parse_config(@google_fluentd_config_path)
    @log.debug(
      'analyze_config plugin: successfully parsed google-fluentd' \
      " configuration file at #{@google_fluentd_config_path}. #{config}"
    )
    baseline_config = parse_config(@google_fluentd_baseline_config_path)
    @log.debug(
      'analyze_config plugin: successfully parsed google-fluentd' \
      ' baseline configuration file at' \
      " #{@google_fluentd_baseline_config_path}: #{baseline_config}"
    )

    # Create hash of all baseline elements by their plugin names.
    baseline_elements = Hash[baseline_config.elements.collect do |e|
                               [default_plugin_name(e), e]
                             end]
    baseline_google_element = baseline_config.elements.find do |e|
      e['@type'] == 'google_cloud'
    end

    # Look at each top-level config element and see whether it
    # matches the baseline value.
    #
    # Note on custom configurations: If the plugin has a custom
    # value (e.g. if a tail plugin has pos_file
    # /var/lib/google-fluentd/pos/my-custom-value.pos), then the
    # default_plugin_name (e.g. source/tail/my-custom-value) won't
    # be a key in baseline_elements below, so it won't be
    # used.  Instead it will use the custom_plugin_name
    # (e.g. source/tail).
    config.elements.each do |e|
      plugin_name = default_plugin_name(e)
      if baseline_elements.key?(plugin_name)
        is_default_plugin = true
        has_default_config = (baseline_elements[plugin_name] == e)
      else
        plugin_name = custom_plugin_name(e)
        is_default_plugin = false
        has_default_config = false
      end
      enabled_plugins_counter.increment(
        labels: {
          plugin_name: plugin_name,
          is_default_plugin: is_default_plugin,
          has_default_config: has_default_config,
          has_ruby_snippet: embedded_ruby?(e)
        },
        by: 1
      )

      # Additional metric for Google plugins (google_cloud and
      # detect_exceptions).
      next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])

      GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
        plugin_config_counter.increment(
          labels: {
            plugin_name: e['@type'],
            param: p,
            is_present: e.key?(p),
            has_default_config: (e.key?(p) &&
                                baseline_google_element.key?(p) &&
                                e[p] == baseline_google_element[p])
          },
          by: 1
        )
        next unless e.key?(p) && %w[true false].include?(e[p])

        config_bool_values_counter.increment(
          labels: {
            plugin_name: e['@type'],
            param: p,
            value: e[p] == 'true'
          },
          by: 1
        )
      end
    end
    @log.info(
      'analyze_config plugin: Successfully finished analyzing config.'
    )
  else
    @log.info(
      'analyze_config plugin: google-fluentd configuration file does not ' \
      "exist at #{@google_fluentd_config_path} or google-fluentd " \
      'baseline configuration file does not exist at' \
      " #{@google_fluentd_baseline_config_path}. Skipping configuration " \
      'analysis.'
    )
  end
rescue StandardError => e
  # Do not crash the agent due to configuration analysis failures.
  @log.warn(
    'analyze_config plugin: Failed to optionally analyze the ' \
    "google-fluentd configuration file. Proceeding anyway. Error: #{e}. " \
    "Trace: #{e.backtrace}"
  )
end
custom_plugin_name(conf_element) click to toggle source

Returns a name for identifying plugins not in our default config. This should not contain arbitrary user-supplied data.

# File lib/fluent/plugin/filter_analyze_config.rb, line 202
def custom_plugin_name(conf_element)
  if KNOWN_PLUGINS.key?(conf_element.name) &&
     KNOWN_PLUGINS[conf_element.name].include?(conf_element['@type'])
    "#{conf_element.name}/#{conf_element['@type']}"
  else
    conf_element.name.to_s
  end
end
default_plugin_name(conf_element) click to toggle source

Returns a name for identifying plugins we ship by default.

# File lib/fluent/plugin/filter_analyze_config.rb, line 189
def default_plugin_name(conf_element)
  case conf_element['@type']
  when 'syslog'
    "#{conf_element.name}/syslog/#{conf_element['protocol_type']}"
  when 'tail'
    "#{conf_element.name}/tail/#{File.basename(conf_element['pos_file'], '.pos')}"
  else
    "#{conf_element.name}/#{conf_element['@type']}"
  end
end
embedded_ruby?(conf_element) click to toggle source
# File lib/fluent/plugin/filter_analyze_config.rb, line 211
def embedded_ruby?(conf_element)
  (conf_element.arg.include?('#{') ||
   conf_element.any? { |_, v| v.include?('#{') } ||
   conf_element.elements.any? { |e| embedded_ruby?(e) })
end
filter(tag, time, record) click to toggle source

rubocop:disable Lint/UnusedMethodArgument

# File lib/fluent/plugin/filter_analyze_config.rb, line 404
def filter(tag, time, record)
  # Skip the actual filtering process.
  record
end
parse_config(path) click to toggle source
# File lib/fluent/plugin/filter_analyze_config.rb, line 173
def parse_config(path)
  data = File.open(path, 'r', &:read)
  fname = File.basename(path)
  basepath = File.dirname(path)
  eval_context = Kernel.binding
  # Override instance_eval so that LiteralParser does not actually
  # evaluate the embedded Ruby, but instead just returns the
  # source string.  See
  # https://github.com/fluent/fluentd/blob/master/lib/fluent/config/literal_parser.rb
  def eval_context.instance_eval(code)
    code
  end
  Fluent::Config::V1Parser.parse(data, fname, basepath, eval_context)
end
shutdown() click to toggle source
Calls superclass method
# File lib/fluent/plugin/filter_analyze_config.rb, line 396
def shutdown
  super
  # Export metrics on shutdown. This is a best-effort attempt, and it might
  # fail, for instance if there was a recent write to the same time series.
  @registry&.export
end
start() click to toggle source

rubocop:enable Style/HashSyntax

Calls superclass method
# File lib/fluent/plugin/filter_analyze_config.rb, line 164
def start
  super
  @log = $log # rubocop:disable Style/GlobalVars

  @log.info(
    'analyze_config plugin: Started the plugin to analyze configuration.'
  )
end