class LogStash::Filters::Date
The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event.
For example, syslog events usually have timestamps like this:
- source,ruby
-
“Apr 17 09:32:01”
You would use the date format ‘MMM dd HH:mm:ss` to parse this.
The date filter is especially important for sorting events and for backfilling old data. If you don’t get the date correct in your event, then searching for them later will likely sort out of order.
In the absence of this filter, logstash will choose a timestamp based on the first time it sees the event (at input time), if the timestamp is not already set in the event. For example, with file input, the timestamp is set to the time of each read.
Constants
- DATEPATTERNS
LOGSTASH-34
- JavaException
- UTC
Public Class Methods
new(config = {})
click to toggle source
Calls superclass method
# File lib/logstash/filters/date.rb, line 101 def initialize(config = {}) super @parsers = Hash.new { |h,k| h[k] = [] } end
Public Instance Methods
filter(event)
click to toggle source
def register
# File lib/logstash/filters/date.rb, line 233 def filter(event) @logger.debug? && @logger.debug("Date filter: received event", :type => event["type"]) @parsers.each do |field, fieldparsers| @logger.debug? && @logger.debug("Date filter looking for field", :type => event["type"], :field => field) next unless event.include?(field) fieldvalues = event[field] fieldvalues = [fieldvalues] if !fieldvalues.is_a?(Array) fieldvalues.each do |value| next if value.nil? begin epochmillis = nil success = false last_exception = RuntimeError.new "Unknown" fieldparsers.each do |parserconfig| parserconfig[:parser].each do |parser| begin if @sprintf_timezone epochmillis = parser.call(value, event.sprintf(@timezone)) else epochmillis = parser.call(value) end success = true break # success rescue StandardError, JavaException => e last_exception = e end end # parserconfig[:parser].each break if success end # fieldparsers.each raise last_exception unless success # Convert joda DateTime to a ruby Time event[@target] = LogStash::Timestamp.at(epochmillis / 1000, (epochmillis % 1000) * 1000) @logger.debug? && @logger.debug("Date parsing done", :value => value, :timestamp => event[@target]) filter_matched(event) rescue StandardError, JavaException => e @logger.warn("Failed parsing date from field", :field => field, :value => value, :exception => e.message, :config_parsers => fieldparsers.collect {|x| x[:format]}.join(','), :config_locale => @locale ? @locale : "default="+java.util.Locale.getDefault().toString() ) # Tag this event if we can't parse it. We can use this later to # reparse+reindex logs if we improve the patterns given. @tag_on_failure.each do |tag| event["tags"] ||= [] event["tags"] << tag unless event["tags"].include?(tag) end end # begin end # fieldvalue.each end # @parsers.each return event end
register()
click to toggle source
# File lib/logstash/filters/date.rb, line 108 def register require "java" if @match.length < 2 raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register", :plugin => "filter", :type => "date", :error => "The match setting should contains first a field name and at least one date format, current value is #{@match}") end locale = nil if @locale if @locale.include? '_' @logger.warn("Date filter now use BCP47 format for locale, replacing underscore with dash") @locale.gsub!('_','-') end locale = java.util.Locale.forLanguageTag(@locale) end @sprintf_timezone = @timezone && !@timezone.index("%{").nil? setupMatcher(@config["match"].shift, locale, @config["match"] ) end
setupMatcher(field, locale, value)
click to toggle source
# File lib/logstash/filters/date.rb, line 130 def setupMatcher(field, locale, value) value.each do |format| parsers = [] case format when "ISO8601" iso_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser if @timezone && !@sprintf_timezone iso_parser = iso_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone)) else iso_parser = iso_parser.withOffsetParsed end parsers << lambda { |date| iso_parser.parseMillis(date) } #Fall back solution of almost ISO8601 date-time almostISOparsers = [ org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSZ").getParser(), org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").getParser(), org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss,SSSZ").getParser(), org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss,SSS").getParser() ].to_java(org.joda.time.format.DateTimeParser) joda_parser = org.joda.time.format.DateTimeFormatterBuilder.new.append( nil, almostISOparsers ).toFormatter() if @timezone && !@sprintf_timezone joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone)) else joda_parser = joda_parser.withOffsetParsed end parsers << lambda { |date| joda_parser.parseMillis(date) } when "UNIX" # unix epoch parsers << lambda do |date| raise "Invalid UNIX epoch value '#{date}'" unless /^\d+(?:\.\d+)?$/ === date || date.is_a?(Numeric) (date.to_f * 1000).to_i end when "UNIX_MS" # unix epoch in ms parsers << lambda do |date| raise "Invalid UNIX epoch value '#{date}'" unless /^\d+$/ === date || date.is_a?(Numeric) date.to_s.slice(0..12).to_i end when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds parsers << lambda do |date| # Skip leading "@" if it is present (common in tai64n times) date = date[1..-1] if date[0, 1] == "@" return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000) end else begin format_has_year = format.match(/y|Y/) joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format) if @timezone && !@sprintf_timezone joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone)) else joda_parser = joda_parser.withOffsetParsed end if locale joda_parser = joda_parser.withLocale(locale) end if @sprintf_timezone parsers << lambda { |date , tz| joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz)).parseMillis(date) } end parsers << lambda do |date| return joda_parser.parseMillis(date) if format_has_year now = Time.now now_month = now.month result = joda_parser.parseDateTime(date) event_month = result.month_of_year.get if (event_month == now_month) result.with_year(now.year) elsif (event_month == 12 && now_month == 1) result.with_year(now.year-1) elsif (event_month == 1 && now_month == 12) result.with_year(now.year+1) else result.with_year(now.year) end.get_millis end #Include a fallback parser to english when default locale is non-english if !locale && "en" != java.util.Locale.getDefault().getLanguage() && (format.include?("MMM") || format.include?("E")) en_joda_parser = joda_parser.withLocale(java.util.Locale.forLanguageTag('en-US')) parsers << lambda { |date| en_joda_parser.parseMillis(date) } end rescue JavaException => e raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register", :plugin => "filter", :type => "date", :error => "#{e.message} for pattern '#{format}'") end end @logger.debug("Adding type with date config", :type => @type, :field => field, :format => format) @parsers[field] << { :parser => parsers, :format => format } end end