class Fluent::Plugin::CloudFoundrySyslogParser
Constants
- CF_IANA_ENTERPRISE_ID
www.iana.org/assignments/enterprise-numbers/enterprise-numbers
- GOROUTER_MESSAGE_EXTRADATA_REGEX
- GOROUTER_MESSAGE_STATIC_REGEX
Regex to extract information from Gorouter access logs github.com/cloudfoundry/gorouter#access-logs <Request Host> - [<Start Date>] “<Request Method> <Request URL> <Request Protocol>” <Status Code> <Bytes Received> <Bytes Sent> “<Referer>” “<User-Agent>” <Remote Address> <Backend Address> x_forwarded_for:“<X-Forwarded-For>” x_forwarded_proto:“<X-Forwarded-Proto>” vcap_request_id:<X-Vcap-Request-ID> response_time:<Response Time> gorouter_time:<Gorouter Time> app_id:<Application ID> app_index:<Application Index> x_cf_routererror:<X-Cf-RouterError> <Extra Headers>
- SYSLOG_HEADER_FIELDS
datatracker.ietf.org/doc/html/rfc5424#section-6.2 PRI is parsed separately (datatracker.ietf.org/doc/html/rfc5424#section-6.2.1)
- SYSLOG_HEADER_SPLIT_CHAR
Syslog Constants datatracker.ietf.org/doc/html/rfc5424#section-6
- SYSLOG_NILVALUE
- SYSLOG_PRI_DELIMITER_END
- SYSLOG_PRI_DELIMITER_START
- SYSLOG_SD_DELIMITER_END
- SYSLOG_SD_DELIMITER_START
- SYSLOG_SD_ID_MATCH_REGEX
Matches the SD-ID if applied to the
SYSLOG_STRUCTURED_DATA_MATCH_REGEX
match- SYSLOG_SD_PARAM_MATCH_REGEX
Matches an SD-PARAM (SD-NAME=SD-VALUE)
- SYSLOG_SEVERITY_CODES
datatracker.ietf.org/doc/html/rfc5424#appendix-A.3 resources.docs.pivotal.io/pdfs/tiledev-guide-2.1.pdf
- SYSLOG_STRUCTURED_DATA_MATCH_REGEX
Regexes to extract information from STRUCTURED-DATA Matches a whole STRUCTURED-DATA block including the delimiters ‘[’, ‘]’
Public Class Methods
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 63 def initialize super end
Public Instance Methods
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 67 def configure(conf) super @time_parser = time_parser_create(format: "%Y-%m-%dT%H:%M:%S.%L%z") end
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 72 def parse(text) if text.nil? or not text.start_with?(SYSLOG_PRI_DELIMITER_START) yield nil return end cursor = 0 record = {} if @include_raw_message record["raw"] = text end # RFC 5424 currently only defines version 1 # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.2 record["header"], cursor = parse_header(text) if cursor.nil? or record.dig("header", "version") != "1" yield nil return end # Convert to integer for convenience record["header"]["version"] = 1 # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3 time = @time_parser.parse(record["header"]["timestamp"]) rescue nil if time.nil? yield nil return end # Parse STRUCTURED_DATA record["sd"], cursor = parse_structured_data(text, cursor) if (cursor.nil?) yield nil return end # Parse MESSAGE msg = text.slice(cursor, text.length - cursor) if msg.nil? record["message"] = nil else record["message"] = msg.strip if @parse_gorouter_access_log and record.dig("sd", "tags@#{CF_IANA_ENTERPRISE_ID}", "origin") == "gorouter" record["gorouter"] = parse_gorouter_access_logs(record["message"]) end end yield time, record end
github.com/cloudfoundry/gorouter#access-logs
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 214 def parse_gorouter_access_logs(msg) r = msg.match(GOROUTER_MESSAGE_STATIC_REGEX) if r.nil? then return end extra_headers = msg[r.to_s.length..-1] r = r.named_captures if extra_headers.nil? then return r end extra_headers.strip.scan(GOROUTER_MESSAGE_EXTRADATA_REGEX) { |match| unless match.length == 2 then next end if match[1].start_with?('"') and match[1].end_with?('"') # Strings r[match[0]] = match[1][1..-2] elsif match[1].match(/^[0-9]+(?:\.[0-9+]+)?$/) # Numbers r[match[0]] = match[1].to_f else r[match[0]] = match[1] end } return r end
datatracker.ietf.org/doc/html/rfc5424#section-6.2
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 151 def parse_header(text) facility, severity, c = parse_pri(text) if (c.nil?) then return end c = c + 1 r = { "pri" => { "facility" => facility, "severity" => severity, }, } SYSLOG_HEADER_FIELDS.each { |field| block, endIdx = parse_header_block(text, c) if block.nil? then return end r[field] = block c = endIdx } return r, c end
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 134 def parse_header_block(text, startIdx) i = text.index(SYSLOG_HEADER_SPLIT_CHAR, startIdx) if i.nil? or i - startIdx < 1 then return end return text.slice(startIdx, i - startIdx), i + 1 end
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 128 def parse_integer(str) return Integer(str || "") rescue ArgumentError return end
datatracker.ietf.org/doc/html/rfc5424#section-6.2.1
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 141 def parse_pri(text) unless text.start_with?(SYSLOG_PRI_DELIMITER_START) then return end endIdx = text.index(SYSLOG_PRI_DELIMITER_END, 1) if endIdx.nil? or endIdx < 2 then return end v_pri = parse_integer(text.slice(1, endIdx - 1)) if v_pri.nil? or v_pri < 0 then return end return v_pri >> 3, SYSLOG_SEVERITY_CODES[v_pri & 0b111], endIdx end
datatracker.ietf.org/doc/html/rfc5424#section-6.3.1
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 174 def parse_sd_element(sd_element) sd_params = {} # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.2 sd_id = sd_element[SYSLOG_SD_ID_MATCH_REGEX] if sd_id.nil? then return end sd_id = sd_id[1..-1] # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.3 sd_element.scan(SYSLOG_SD_PARAM_MATCH_REGEX).each { |match| arr = match[0].strip.split("=", 2) sd_params[arr[0]] = arr[1][1..-2] } return sd_id, sd_params end
datatracker.ietf.org/doc/html/rfc5424#section-6.3
# File lib/fluent/plugin/parser_cloudfoundry_syslog.rb, line 192 def parse_structured_data(text, startIdx) if text[startIdx] == SYSLOG_NILVALUE then return {}, startIdx + 1 end unless text[startIdx] == SYSLOG_SD_DELIMITER_START then return end sd = text[startIdx..-1][SYSLOG_STRUCTURED_DATA_MATCH_REGEX] if sd.nil? then return end r = {} len = 0 loop do len += sd.length sd_id, sd_params = parse_sd_element(sd) if sd_id.nil? then return end r[sd_id] = sd_params sd = text[startIdx + len..-1][SYSLOG_STRUCTURED_DATA_MATCH_REGEX] break if sd.nil? end return r, startIdx + len end