class LogStash::Filters::Metrics

The metrics filter is useful for aggregating metrics.

IMPORTANT: Elasticsearch 2.0 no longer allows field names with dots. Version 3.0 of the metrics filter plugin changes behavior to use nested fields rather than dotted notation to avoid colliding with versions of Elasticsearch 2.0+. Please note the changes in the documentation (underscores and sub-fields used).

For example, if you have a field `response` that is a http response code, and you want to count each kind of response, you can do this:

source,ruby

filter {

metrics {
  meter => [ "http_%{response}" ]
  add_tag => "metric"
}

}

Metrics are flushed every 5 seconds by default or according to `flush_interval`. Metrics appear as new events in the event stream and go through any filters that occur after as well as outputs.

In general, you will want to add a tag to your metrics and have an output explicitly look for that tag.

The event that is flushed will include every 'meter' and 'timer' metric in the following way:

`meter` values

For a `meter => “thing”` you will receive the following fields:

`timer` values

For a `timer => [ “thing”, “%{duration}” ]` you will receive the following fields:

The default lengths of the event rate window (1, 5, and 15 minutes) can be configured with the `rates` option.

Example: Computing event rate

For a simple example, let's track how many events per second are running through logstash:

source,ruby

input {
  generator {
    type => "generated"
  }
}

filter {
  if [type] == "generated" {
    metrics {
      meter => "events"
      add_tag => "metric"
    }
  }
}

output {
  # only emit events with the 'metric' tag
  if "metric" in [tags] {
    stdout {
      codec => line {
        format => "rate: %{[events][rate_1m]}"
      }
    }
  }
}

Running the above:

source,ruby

% bin/logstash -f example.conf rate: 23721.983566819246 rate: 24811.395722536377 rate: 25875.892745934525 rate: 26836.42375967113

We see the output includes our events' 1-minute rate.

In the real world, you would emit this to graphite or another metrics store, like so:

source,ruby

output {

graphite {
  metrics => [ "events.rate_1m", "%{[events][rate_1m]}" ]
}

}

Public Instance Methods

filter(event) click to toggle source
# File lib/logstash/filters/metrics.rb, line 170
def filter(event)


  # TODO(piavlo): This should probably be moved to base filter class.
  if @ignore_older_than > 0 && Time.now - event.timestamp.time > @ignore_older_than
    @logger.debug("Skipping metriks for old event", :event => event)
    return
  end

  @meter.each do |m|
    @metric_meters[event.sprintf(m)].mark
  end

  @timer.each do |name, value|
    @metric_timers[event.sprintf(name)].update(event.sprintf(value).to_f)
  end
end
flush(options = {}) click to toggle source
# File lib/logstash/filters/metrics.rb, line 188
def flush(options = {})
  # Add 5 seconds to @last_flush and @last_clear counters
  # since this method is called every 5 seconds.
  @last_flush.update { |v| v + 5 }
  @last_clear.update { |v| v + 5 }

  # Do nothing if there's nothing to do ;)
  return unless should_flush?

  event = LogStash::Event.new
  event.set("message", @host)
  @metric_meters.each_pair do |name, metric|
    flush_rates event, name, metric
    metric.clear if should_clear?
  end

  @metric_timers.each_pair do |name, metric|
    flush_rates event, name, metric
    # These 4 values are not sliding, so they probably are not useful.
    event.set("[#{name}][min]", metric.min)
    event.set("[#{name}][max]", metric.max)
    # timer's stddev currently returns variance, fix it.
    event.set("[#{name}][stddev]", metric.stddev ** 0.5)
    event.set("[#{name}][mean]", metric.mean)

    @percentiles.each do |percentile|
      event.set("[#{name}][p#{percentile}]", metric.snapshot.value(percentile / 100.0))
    end
    metric.clear if should_clear?
  end

  # Reset counter since metrics were flushed
  @last_flush.value = 0

  if should_clear?
    #Reset counter since metrics were cleared
    @last_clear.value = 0
    @metric_meters.clear
    @metric_timers.clear
  end

  filter_matched(event)
  return [event]
end
periodic_flush() click to toggle source

this is a temporary fix to enable periodic flushes without using the plugin config:

config :periodic_flush, :validate => :boolean, :default => true

because this is not optional here and should not be configurable. this is until we refactor the periodic_flush mechanism per github.com/elasticsearch/logstash/issues/1839

# File lib/logstash/filters/metrics.rb, line 238
def periodic_flush
  true
end
register() click to toggle source
# File lib/logstash/filters/metrics.rb, line 153
def register
  require "metriks"
  require "socket"
  require "atomic"
  require "thread_safe"
  @last_flush = Atomic.new(0) # how many seconds ago the metrics where flushed.
  @last_clear = Atomic.new(0) # how many seconds ago the metrics where cleared.
  @random_key_preffix = SecureRandom.hex
  # Same as logstash-input-file
  @host = Socket.gethostname.force_encoding(Encoding::UTF_8)
  unless (@rates - [1, 5, 15]).empty?
    raise LogStash::ConfigurationError, "Invalid rates configuration. possible rates are 1, 5, 15. Rates: #{rates}."
  end
  @metric_meters = ThreadSafe::Cache.new { |h,k| h[k] = Metriks.meter metric_key(k) }
  @metric_timers = ThreadSafe::Cache.new { |h,k| h[k] = Metriks.timer metric_key(k) }
end

Private Instance Methods

flush_rates(event, name, metric) click to toggle source
# File lib/logstash/filters/metrics.rb, line 244
def flush_rates(event, name, metric)
    event.set("[#{name}][count]", metric.count)
    event.set("[#{name}][rate_1m]", metric.one_minute_rate) if @rates.include? 1
    event.set("[#{name}][rate_5m]", metric.five_minute_rate) if @rates.include? 5
    event.set("[#{name}][rate_15m]", metric.fifteen_minute_rate) if @rates.include? 15
end
metric_key(key) click to toggle source
# File lib/logstash/filters/metrics.rb, line 251
def metric_key(key)
  "#{@random_key_preffix}_#{key}"
end
should_clear?() click to toggle source
# File lib/logstash/filters/metrics.rb, line 259
def should_clear?
  @clear_interval > 0 && @last_clear.value >= @clear_interval
end
should_flush?() click to toggle source
# File lib/logstash/filters/metrics.rb, line 255
def should_flush?
  @last_flush.value >= @flush_interval && (!@metric_meters.empty? || !@metric_timers.empty?)
end