Skip to content

Commit db71a89

Browse files
cosmo0920cosmo0920
committed
Backport fluent#1492 to v0.12
1 parent 2c83da3 commit db71a89

File tree

2 files changed

+203
-0
lines changed

2 files changed

+203
-0
lines changed

‎lib/fluent/parser.rb

+41
Original file line numberDiff line numberDiff line change
@@ -536,9 +536,13 @@ class SyslogParser < Parser
536536
REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
537537
# From in_syslog default pattern
538538
REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
539+
REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
540+
REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
539541

540542
config_param :time_format, :string, default: "%b %d %H:%M:%S"
541543
config_param :with_priority, :bool, default: false
544+
config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
545+
config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
542546

543547
def initialize
544548
super
@@ -549,6 +553,26 @@ def configure(conf)
549553
super
550554

551555
@regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
556+
@time_parser_rfc3164 = @time_parser_rfc5424 = nil
557+
@regexp = case @message_format
558+
when :rfc3164
559+
class << self
560+
alias_method :parse, :parse_plain
561+
end
562+
@with_priority ? REGEXP_WITH_PRI : REGEXP
563+
when :rfc5424
564+
class << self
565+
alias_method :parse, :parse_plain
566+
end
567+
REGEXP_RFC5424
568+
when :auto
569+
class << self
570+
alias_method :parse, :parse_auto
571+
end
572+
@time_parser_rfc3164 = TextParser::TimeParser.new(@time_format)
573+
@time_parser_rfc5424 = TextParser::TimeParser.new(@rfc5424_time_format)
574+
nil
575+
end
552576
@time_parser = TextParser::TimeParser.new(@time_format)
553577
end
554578

@@ -557,6 +581,23 @@ def patterns
557581
end
558582

559583
def parse(text)
584+
# This is overwritten in configure
585+
end
586+
587+
def parse_auto(text, &block)
588+
if @message_format == :auto
589+
if REGEXP_DETECT_RFC5424.match(text)
590+
@regexp = REGEXP_RFC5424
591+
@time_parser = @time_parser_rfc5424
592+
else
593+
@regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
594+
@time_parser = @time_parser_rfc3164
595+
end
596+
end
597+
parse_plain(text, &block)
598+
end
599+
600+
def parse_plain(text, &block)
560601
m = @regexp.match(text)
561602
unless m
562603
if block_given?

‎test/test_parser.rb

+162
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,168 @@ def test_parse_with_keep_time_key
364364
assert_equal "Feb 28 00:00:12", record['time']
365365
end
366366
end
367+
368+
class TestRFC5424Regexp < self
369+
def test_parse_with_rfc5424_message
370+
@parser.configure(
371+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
372+
'message_format' => 'rfc5424',
373+
)
374+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
375+
@parser.parse(text) do |time, record|
376+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
377+
assert_equal "-", record["pid"]
378+
assert_equal "-", record["msgid"]
379+
assert_equal "-", record["extradata"]
380+
assert_equal "Hi, from Fluentd!", record["message"]
381+
end
382+
end
383+
384+
def test_parse_with_rfc5424_structured_message
385+
@parser.configure(
386+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
387+
'message_format' => 'rfc5424',
388+
)
389+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
390+
@parser.parse(text) do |time, record|
391+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
392+
assert_equal "11111", record["pid"]
393+
assert_equal "ID24224", record["msgid"]
394+
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
395+
record["extradata"]
396+
assert_equal "Hi, from Fluentd!", record["message"]
397+
end
398+
end
399+
end
400+
401+
class TestAutoRegexp < self
402+
def test_auto_with_legacy_syslog_message
403+
@parser.configure(
404+
'time_format' => '%b %d %M:%S:%H',
405+
'mseeage_format' => 'auto',
406+
)
407+
text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
408+
@parser.parse(text) do |time, record|
409+
assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
410+
assert_equal(@expected, record)
411+
end
412+
end
413+
414+
def test_auto_with_legacy_syslog_priority_message
415+
@parser.configure(
416+
'time_format' => '%b %d %M:%S:%H',
417+
'with_priority' => true,
418+
'mseeage_format' => 'auto',
419+
)
420+
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
421+
@parser.parse(text) do |time, record|
422+
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
423+
assert_equal(@expected.merge('pri' => 6), record)
424+
end
425+
end
426+
427+
def test_parse_with_rfc5424_message
428+
@parser.configure(
429+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
430+
'message_format' => 'auto',
431+
)
432+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
433+
@parser.parse(text) do |time, record|
434+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
435+
assert_equal "-", record["pid"]
436+
assert_equal "-", record["msgid"]
437+
assert_equal "-", record["extradata"]
438+
assert_equal "Hi, from Fluentd!", record["message"]
439+
end
440+
end
441+
442+
def test_parse_with_rfc5424_structured_message
443+
@parser.configure(
444+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
445+
'message_format' => 'auto',
446+
)
447+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
448+
@parser.parse(text) do |time, record|
449+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
450+
assert_equal "11111", record["pid"]
451+
assert_equal "ID24224", record["msgid"]
452+
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
453+
record["extradata"]
454+
assert_equal "Hi, from Fluentd!", record["message"]
455+
end
456+
end
457+
458+
def test_parse_with_both_message_type
459+
@parser.configure(
460+
'time_format' => '%b %d %M:%S:%H',
461+
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
462+
'message_format' => 'auto',
463+
)
464+
text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
465+
@parser.parse(text) do |time, record|
466+
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
467+
assert_equal(@expected, record)
468+
end
469+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
470+
@parser.parse(text) do |time, record|
471+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
472+
assert_equal "11111", record["pid"]
473+
assert_equal "ID24224", record["msgid"]
474+
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
475+
record["extradata"]
476+
assert_equal "Hi, from Fluentd!", record["message"]
477+
end
478+
text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
479+
@parser.parse(text) do |time, record|
480+
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
481+
assert_equal(@expected, record)
482+
end
483+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
484+
@parser.parse(text) do |time, record|
485+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
486+
assert_equal "-", record["pid"]
487+
assert_equal "-", record["msgid"]
488+
assert_equal "-", record["extradata"]
489+
assert_equal "Hi, from Fluentd!", record["message"]
490+
end
491+
end
492+
493+
def test_parse_with_both_message_type_and_priority
494+
@parser.configure(
495+
'time_format' => '%b %d %M:%S:%H',
496+
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
497+
'with_priority' => true,
498+
'message_format' => 'auto',
499+
)
500+
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
501+
@parser.parse(text) do |time, record|
502+
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
503+
assert_equal(@expected.merge('pri' => 6), record)
504+
end
505+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
506+
@parser.parse(text) do |time, record|
507+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
508+
assert_equal "11111", record["pid"]
509+
assert_equal "ID24224", record["msgid"]
510+
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
511+
record["extradata"]
512+
assert_equal "Hi, from Fluentd!", record["message"]
513+
end
514+
text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
515+
@parser.parse(text) do |time, record|
516+
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
517+
assert_equal(@expected.merge('pri' => 16), record)
518+
end
519+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
520+
@parser.parse(text) do |time, record|
521+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
522+
assert_equal "-", record["pid"]
523+
assert_equal "-", record["msgid"]
524+
assert_equal "-", record["extradata"]
525+
assert_equal "Hi, from Fluentd!", record["message"]
526+
end
527+
end
528+
end
367529
end
368530

369531
class JsonParserTest < ::Test::Unit::TestCase

0 commit comments

Comments
 (0)