Skip to content

Commit

Permalink
parser_syslog: Implement new parser for syslog rfc3164
Browse files Browse the repository at this point in the history 
Signed-off-by: Masahiro Nakagawa <[email protected]>
  • Loading branch information
@repeatedly
repeatedly committed Sep 3, 2019
1 parent c7a825c commit 68f3ec7
Show file tree
Hide file tree
Showing 2 changed files with 143 additions and 22 deletions.
107 changes: 104 additions & 3 deletions lib/fluent/plugin/parser_syslog.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ class SyslogParser < Parser
config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
desc 'Specify time format for event time for rfc5424 protocol'
config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
desc 'The parser type used to parse syslog message'
config_param :parser_type, :enum, list: [:regexp, :string], default: :regexp
desc 'support colonless ident in string parser'
config_param :support_colonless_ident, :bool, default: true

def initialize
super
Expand All @@ -50,10 +54,17 @@ def configure(conf)
@time_parser_rfc3164 = @time_parser_rfc5424 = nil
@time_parser_rfc5424_without_subseconds = nil
@support_rfc5424_without_subseconds = false
@regexp_parser = @parser_type == :regexp
@regexp = case @message_format
when :rfc3164
class << self
alias_method :parse, :parse_plain
if @regexp_parser
class << self
alias_method :parse, :parse_plain
end
else
class << self
alias_method :parse, :parse_rfc3164
end
end
@with_priority ? REGEXP_WITH_PRI : REGEXP
when :rfc5424
Expand Down Expand Up @@ -88,11 +99,16 @@ def parse_auto(text, &block)
@regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
@time_parser = @time_parser_rfc5424
@support_rfc5424_without_subseconds = true
parse_plain(text, &block)
else
@regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
@time_parser = @time_parser_rfc3164
if @regexp_parser
parse_plain(text, &block)
else
parse_rfc3164(text, &block)
end
end
parse_plain(text, &block)
end

def parse_plain(text, &block)
Expand Down Expand Up @@ -137,6 +153,91 @@ def parse_plain(text, &block)

yield time, record
end

SPLIT_CHAR = ' '.freeze
PRI_START_CHAR = '<'.freeze

def parse_rfc3164(text, &block)
pri = nil
start = 0
if @with_priority
if text.start_with?(PRI_START_CHAR)
i = text.index('>'.freeze, 1)
pri = text.slice(1, i - 1).to_i
start = i + 1
else
yield nil, nil
return
end
end

# header part
diff = 15 # skip Mmm dd hh:mm:ss
time_end = text[start + diff]
if time_end == SPLIT_CHAR
time_str = text.slice(start, diff)
start += 16 # time + ' '
elsif time_end == '.'.freeze
# support subsecond time
i = text.index(SPLIT_CHAR, diff)
time_str = text.slice(start, i - start)
start = i + 1
else
yield nil, nil
return
end

i = text.index(SPLIT_CHAR, start)
if i.nil?
yield nil, nil
return
end
diff = i - start
host = text.slice(start, diff)
start += (diff + 1)

i = text.index(SPLIT_CHAR, start)
if i.nil?
yield nil, nil
return
end
diff = i - start

record = {'host' => host}
record['pri'] = pri if pri

# message part
msg = if text[i - 1] == ':'.freeze
if text[i - 2] == ']'.freeze
j = text.index('['.freeze, start)
record['ident'] = text.slice(start, j - start)
record['pid'] = text.slice(j + 1, i - j - 3) # remove '[' / ']:'
else
record['ident'] = text.slice(start, i - start - 1)
end
text.slice(i + 1, text.bytesize)
else
if @support_colonless_ident
if text[i - 1] == ']'.freeze
j = text.index('['.freeze, start)
record['ident'] = text.slice(start, j - start)
record['pid'] = text.slice(j + 1, i - j - 2) # remove '[' / ']'
else
record['ident'] = text.slice(start, i - start)
end
text.slice(i + 1, text.bytesize)
else
text.slice(i - diff, text.bytesize)
end
end
msg.chomp!
record['message'] = msg

time = @time_parser.parse(time_str.squeeze(SPLIT_CHAR))
record['time'] = time_str if @keep_time_key

yield time, record
end
end
end
end
58 changes: 39 additions & 19 deletions test/plugin/test_parser_syslog.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@ def setup
}
end

def test_parse
@parser.configure({})
data('regexp' => 'regexp', 'string' => 'string')
def test_parse(param)
@parser.configure({'parser_type' => param})
@parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
assert_equal(@expected, record)
Expand All @@ -24,17 +25,19 @@ def test_parse
assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
end

def test_parse_with_time_format
@parser.configure('time_format' => '%b %d %M:%S:%H')
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_with_time_format(param)
@parser.configure('time_format' => '%b %d %M:%S:%H', 'parser_type' => param)
@parser.instance.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
assert_equal(@expected, record)
}
assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
end

def test_parse_with_priority
@parser.configure('with_priority' => true)
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_with_priority(param)
@parser.configure('with_priority' => true, 'parser_type' => param)
@parser.instance.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
assert_equal(@expected.merge('pri' => 6), record)
Expand All @@ -43,8 +46,9 @@ def test_parse_with_priority
assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
end

def test_parse_without_colon
@parser.configure({})
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_without_colon(param)
@parser.configure({'parser_type' => param})
@parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
assert_equal(@expected, record)
Expand All @@ -53,29 +57,33 @@ def test_parse_without_colon
assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
end

def test_parse_with_keep_time_key
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_with_keep_time_key(param)
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'keep_time_key'=>'true',
'parser_type' => param
)
text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal "Feb 28 00:00:12", record['time']
end
end

def test_parse_various_characters_for_tag
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_various_characters_for_tag(param)
ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
@parser.configure({})
@parser.configure({'parser_type' => param})
@parser.instance.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
assert_equal(@expected.merge('ident' => ident), record)
}
end

def test_parse_various_characters_for_tag_with_priority
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_various_characters_for_tag_with_priority(param)
ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
@parser.configure('with_priority' => true)
@parser.configure('with_priority' => true, 'parser_type' => param)
@parser.instance.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
Expand Down Expand Up @@ -273,10 +281,12 @@ def test_parse_with_rfc5424_message_both_timestamp
end

class TestAutoRegexp < self
def test_auto_with_legacy_syslog_message
data('regexp' => 'regexp', 'string' => 'string')
def test_auto_with_legacy_syslog_message(param)
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'message_format' => 'auto',
'parser_type' => param
)
text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
Expand All @@ -286,11 +296,13 @@ def test_auto_with_legacy_syslog_message
assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
end

def test_auto_with_legacy_syslog_priority_message
data('regexp' => 'regexp', 'string' => 'string')
def test_auto_with_legacy_syslog_priority_message(param)
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'with_priority' => true,
'message_format' => 'auto',
'parser_type' => param
)
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
Expand All @@ -300,11 +312,13 @@ def test_auto_with_legacy_syslog_priority_message
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
end

def test_parse_with_rfc5424_message
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_with_rfc5424_message(param)
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'auto',
'with_priority' => true,
'parser_type' => param
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
Expand All @@ -318,11 +332,13 @@ def test_parse_with_rfc5424_message
@parser.instance.patterns['format'])
end

def test_parse_with_rfc5424_structured_message
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_with_rfc5424_structured_message(param)
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'auto',
'with_priority' => true,
'parser_type' => param
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
Expand All @@ -337,12 +353,14 @@ def test_parse_with_rfc5424_structured_message
@parser.instance.patterns['format'])
end

def test_parse_with_both_message_type
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_with_both_message_type(param)
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'auto',
'with_priority' => true,
'parser_type' => param
)
text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
Expand Down Expand Up @@ -382,12 +400,14 @@ def test_parse_with_both_message_type
@parser.instance.patterns['format'])
end

def test_parse_with_both_message_type_and_priority
data('regexp' => 'regexp', 'string' => 'string')
def test_parse_with_both_message_type_and_priority(param)
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'with_priority' => true,
'message_format' => 'auto',
'parser_type' => param
)
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
Expand Down

0 comments on commit 68f3ec7

Please sign in to comment.