Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support rfc5424 syslog format #1492

Merged
merged 11 commits into from
Mar 8, 2017
Merged

Support rfc5424 syslog format #1492

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
38830b7
parser_syslog: Add rfc5424 regexp
cosmo0920 Mar 7, 2017
9f9f747
Add native implementation for detecting rfc3164 and rfc5424 format
cosmo0920 Mar 7, 2017
84ea64c
Use parsed time in assertions instead of kept time record
cosmo0920 Mar 7, 2017
0918deb
Use more strict regex for rfc5424
cosmo0920 Mar 7, 2017
f36a662
Loosen regexp in extradata part
cosmo0920 Mar 7, 2017
b0e9b73
Implement more flexible syslog format detection
cosmo0920 Mar 7, 2017
b33d743
Re-create time_parser when auto choosed
cosmo0920 Mar 8, 2017
ac1496d
Use more complex test case for auto format detection test
cosmo0920 Mar 8, 2017
d71c39b
Forward appropriate methods
cosmo0920 Mar 8, 2017
5ba24ad
Create time parsers in #configure phase
cosmo0920 Mar 8, 2017
866b2ca
Keep #parse for method check safety
cosmo0920 Mar 8, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 35 additions & 2 deletions lib/fluent/plugin/parser_syslog.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,13 @@ class SyslogParser < Parser
REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
# From in_syslog default pattern
REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/

config_set_default :time_format, "%b %d %H:%M:%S"
config_param :with_priority, :bool, default: false
config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"

def initialize
super
Expand All @@ -39,15 +43,44 @@ def initialize
def configure(conf)
super

@regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
@regexp = case @message_format
when :rfc3164
class << self
alias_method :parse, :parse_plain
end
Copy link
Member

@repeatedly repeatedly Mar 8, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Keep parse(text) for method check safety.

def parse(text)
  # This is overwritten in configure
end

@with_priority ? REGEXP_WITH_PRI : REGEXP
when :rfc5424
class << self
alias_method :parse, :parse_plain
end
REGEXP_RFC5424
when :auto
class << self
alias_method :parse, :parse_auto
end
nil
end
@time_parser = time_parser_create
end

def patterns
{'format' => @regexp, 'time_format' => @time_format}
end

def parse(text)
def parse_auto(text, &block)
if @message_format == :auto
if REGEXP_DETECT_RFC5424.match(text)
@regexp = REGEXP_RFC5424
@time_parser = time_parser_create(format: @rfc5424_time_format)
else
@regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
@time_parser = time_parser_create(format: @time_format)
end
end
parse_plain(text, &block)
end

def parse_plain(text, &block)
m = @regexp.match(text)
unless m
yield nil, nil
Expand Down
162 changes: 162 additions & 0 deletions test/plugin/test_parser_syslog.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,166 @@ def test_parse_with_keep_time_key
assert_equal "Feb 28 00:00:12", record['time']
end
end

class TestRFC5424Regexp < self
def test_parse_with_rfc5424_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'rfc5424',
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "-", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_rfc5424_structured_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'rfc5424',
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end
end

class TestAutoRegexp < self
def test_auto_with_legacy_syslog_message
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'mseeage_format' => 'auto',
)
text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected, record)
end
end

def test_auto_with_legacy_syslog_priority_message
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'with_priority' => true,
'mseeage_format' => 'auto',
)
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected.merge('pri' => 6), record)
end
end

def test_parse_with_rfc5424_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'auto',
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "-", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_rfc5424_structured_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'auto',
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_both_message_type
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'auto',
)
text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected, record)
end
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected, record)
end
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "-", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_both_message_type_and_priority
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'with_priority' => true,
'message_format' => 'auto',
)
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected.merge('pri' => 6), record)
end
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected.merge('pri' => 16), record)
end
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "-", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end
end
end