From 06478df8ff75498aa0c66817b2a416d8adbda6d2 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Thu, 16 Jun 2016 19:57:12 +0900 Subject: [PATCH 01/21] add empty? method to all event stream classes (to solve API inconsistency) --- lib/fluent/event.rb | 10 ++++++++++ test/test_event.rb | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/lib/fluent/event.rb b/lib/fluent/event.rb index f4ac6974dd..ff1d18a921 100644 --- a/lib/fluent/event.rb +++ b/lib/fluent/event.rb @@ -26,6 +26,10 @@ def size end alias :length :size + def empty? + size == 0 + end + def repeatable? false end @@ -166,6 +170,12 @@ def initialize(data, cached_unpacker = nil, size = 0) @size = size end + def empty? + # This is not correct, but actual number of records will be shown after iteration, and + # "size" argument is always 0 currently (because forward protocol doesn't tell it to destination) + false + end + def size @size end diff --git a/test/test_event.rb b/test/test_event.rb index 1d8d6332bd..def49c9a4a 100644 --- a/test/test_event.rb +++ b/test/test_event.rb @@ -12,6 +12,10 @@ def setup @es = OneEventStream.new(@time, @record) end + test 'empty?' do + assert_false @es.empty? + end + test 'repeatable?' do assert_true @es.repeatable? end @@ -153,6 +157,10 @@ def setup @es = MessagePackEventStream.new(pk.to_s) end + test 'empty?' do + assert_false @es.empty? + end + test 'repeatable?' do assert_true @es.repeatable? end From 98b9959b19d62121735291f522c72d4f05fffdfd Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Fri, 17 Jun 2016 16:47:21 +0900 Subject: [PATCH 02/21] Make it sure to implement #dup at all event stream classes, and fix bug not to dup correctly --- lib/fluent/event.rb | 23 ++++++++++++++--------- test/test_event.rb | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/lib/fluent/event.rb b/lib/fluent/event.rb index ff1d18a921..042b813a54 100644 --- a/lib/fluent/event.rb +++ b/lib/fluent/event.rb @@ -21,6 +21,11 @@ class EventStream include Enumerable include MessagePackFactory::Mixin + # dup does deep copy for event stream + def dup + raise NotImplementedError, "DO NOT USE THIS CLASS directly." + end + def size raise NotImplementedError, "DO NOT USE THIS CLASS directly." end @@ -90,7 +95,7 @@ def initialize(entries) end def dup - entries = @entries.map { |entry| entry.dup } # @entries.map(:dup) doesn't work by ArgumentError + entries = @entries.map{ |time, record| [time, record.dup] } ArrayEventStream.new(entries) end @@ -123,17 +128,13 @@ def each(&block) # 2. add events # stream[tag].add(time, record) class MultiEventStream < EventStream - def initialize - @time_array = [] - @record_array = [] + def initialize(time_array = [], record_array = []) + @time_array = time_array + @record_array = record_array end def dup - es = MultiEventStream.new - @time_array.zip(@record_array).each { |time, record| - es.add(time, record.dup) - } - es + MultiEventStream.new(@time_array.dup, @record_array.map(&:dup)) end def size @@ -176,6 +177,10 @@ def empty? false end + def dup + MessagePackEventStream.new(@data.dup, @size) + end + def size @size end diff --git a/test/test_event.rb b/test/test_event.rb index def49c9a4a..5a3520cdbc 100644 --- a/test/test_event.rb +++ b/test/test_event.rb @@ -3,8 +3,26 @@ require 'fluent/event' module EventTest + module DeepCopyAssertion + def assert_duplicated_records(es1, es2) + ary1 = [] + es1.each do |_, record| + ary1 << record + end + ary2 = [] + es2.each do |_, record| + ary2 << record + end + assert_equal ary1.size, ary2.size + ary1.each_with_index do |r, i| + assert_not_equal r.object_id, ary2[i].object_id + end + end + end + class OneEventStreamTest < ::Test::Unit::TestCase include Fluent + include DeepCopyAssertion def setup @time = event_time() @@ -16,6 +34,10 @@ def setup assert_false @es.empty? end + test 'size' do + assert_equal 1, @es.size + end + test 'repeatable?' do assert_true @es.repeatable? end @@ -24,6 +46,7 @@ def setup dupped = @es.dup assert_kind_of OneEventStream, dupped assert_not_equal @es.object_id, dupped.object_id + assert_duplicated_records @es, dupped end test 'each' do @@ -52,6 +75,7 @@ def setup class ArrayEventStreamTest < ::Test::Unit::TestCase include Fluent + include DeepCopyAssertion def setup time = Engine.now @@ -68,6 +92,7 @@ def setup dupped = @es.dup assert_kind_of ArrayEventStream, dupped assert_not_equal @es.object_id, dupped.object_id + assert_duplicated_records @es, dupped end test 'empty?' do @@ -97,6 +122,7 @@ def setup class MultiEventStreamTest < ::Test::Unit::TestCase include Fluent + include DeepCopyAssertion def setup time = Engine.now @@ -116,6 +142,7 @@ def setup dupped = @es.dup assert_kind_of MultiEventStream, dupped assert_not_equal @es.object_id, dupped.object_id + assert_duplicated_records @es, dupped end test 'empty?' do @@ -145,6 +172,7 @@ def setup class MessagePackEventStreamTest < ::Test::Unit::TestCase include Fluent + include DeepCopyAssertion def setup pk = Fluent::Engine.msgpack_factory.packer @@ -157,6 +185,13 @@ def setup @es = MessagePackEventStream.new(pk.to_s) end + test 'dup' do + dupped = @es.dup + assert_kind_of MessagePackEventStream, dupped + assert_not_equal @es.object_id, dupped.object_id + assert_duplicated_records @es, dupped + end + test 'empty?' do assert_false @es.empty? end From 099a22a296e157dc63264a437f86bf9ec841fd53 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 27 Jun 2016 12:58:31 +0900 Subject: [PATCH 03/21] fix to return empty slice if arguments specifies so --- lib/fluent/event.rb | 8 ++++++++ test/test_event.rb | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/fluent/event.rb b/lib/fluent/event.rb index 042b813a54..4223b84fce 100644 --- a/lib/fluent/event.rb +++ b/lib/fluent/event.rb @@ -79,6 +79,14 @@ def repeatable? true end + def slice(index, num) + if index > 0 || num == 0 + ArrayEventStream.new([]) + else + self.dup + end + end + def each(&block) block.call(@time, @record) nil diff --git a/test/test_event.rb b/test/test_event.rb index 5a3520cdbc..fc5f301d8d 100644 --- a/test/test_event.rb +++ b/test/test_event.rb @@ -49,6 +49,20 @@ def setup assert_duplicated_records @es, dupped end + test 'slice' do + assert_equal 0, @es.slice(1,1).size + assert_equal 0, @es.slice(0,0).size + + sliced = @es.slice(0, 1) + assert_kind_of EventStream, sliced + assert_equal 1, sliced.size + + sliced.each do |time, record| + assert_equal @time, time + assert_equal @record, record + end + end + test 'each' do @es.each { |time, record| assert_equal @time, time From 383ff9be654b595f7265f012670bcedd48b7e2d9 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 21 Jun 2016 15:36:16 +0900 Subject: [PATCH 04/21] move TimeFormatter to Fluent top level in time.rb --- lib/fluent/compat/time_formatter.rb | 114 ---------------------------- lib/fluent/mixin.rb | 6 +- lib/fluent/time.rb | 93 +++++++++++++++++++++++ 3 files changed, 97 insertions(+), 116 deletions(-) delete mode 100644 lib/fluent/compat/time_formatter.rb diff --git a/lib/fluent/compat/time_formatter.rb b/lib/fluent/compat/time_formatter.rb deleted file mode 100644 index a788fd25b3..0000000000 --- a/lib/fluent/compat/time_formatter.rb +++ /dev/null @@ -1,114 +0,0 @@ -# -# Fluentd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'fluent/timezone' -require 'fluent/time' - -module Fluent - module Compat - class TimeFormatter - def initialize(format, localtime, timezone = nil) - @tc1 = 0 - @tc1_str = nil - @tc2 = 0 - @tc2_str = nil - - if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/ - define_singleton_method(:format) {|time| - format_with_subsec(time) - } - else - define_singleton_method(:format) {|time| - format_without_subsec(time) - } - end - - if formatter = Fluent::Timezone.formatter(timezone, format) - define_singleton_method(:format_nocache) {|time| - formatter.call(time) - } - return - end - - if format - if localtime - define_singleton_method(:format_nocache) {|time| - Time.at(time).strftime(format) - } - else - define_singleton_method(:format_nocache) {|time| - Time.at(time).utc.strftime(format) - } - end - else - if localtime - define_singleton_method(:format_nocache) {|time| - Time.at(time).iso8601 - } - else - define_singleton_method(:format_nocache) {|time| - Time.at(time).utc.iso8601 - } - end - end - end - - def format_without_subsec(time) - if @tc1 == time - return @tc1_str - elsif @tc2 == time - return @tc2_str - else - str = format_nocache(time) - if @tc1 < @tc2 - @tc1 = time - @tc1_str = str - else - @tc2 = time - @tc2_str = str - end - return str - end - end - - def format_with_subsec(time) - if Fluent::EventTime.eq?(@tc1, time) - return @tc1_str - elsif Fluent::EventTime.eq?(@tc2, time) - return @tc2_str - else - str = format_nocache(time) - if @tc1 < @tc2 - @tc1 = time - @tc1_str = str - else - @tc2 = time - @tc2_str = str - end - return str - end - end - - def format(time) - # will be overridden in initialize - end - - def format_nocache(time) - # will be overridden in initialize - end - end - end -end diff --git a/lib/fluent/mixin.rb b/lib/fluent/mixin.rb index 0913ef0b27..6d9d694d6a 100644 --- a/lib/fluent/mixin.rb +++ b/lib/fluent/mixin.rb @@ -18,14 +18,16 @@ require 'fluent/compat/handle_tag_name_mixin' require 'fluent/compat/set_time_key_mixin' require 'fluent/compat/set_tag_key_mixin' -require 'fluent/compat/time_formatter' require 'fluent/compat/type_converter' +require 'fluent/time' + module Fluent - TimeFormatter = Fluent::Compat::TimeFormatter RecordFilterMixin = Fluent::Compat::RecordFilterMixin HandleTagNameMixin = Fluent::Compat::HandleTagNameMixin SetTimeKeyMixin = Fluent::Compat::SetTimeKeyMixin SetTagKeyMixin = Fluent::Compat::SetTagKeyMixin TypeConverter = Fluent::Compat::TypeConverter + + TimeFormatter = Fluent::TimeFormatter end diff --git a/lib/fluent/time.rb b/lib/fluent/time.rb index 1914d63f91..0852923374 100644 --- a/lib/fluent/time.rb +++ b/lib/fluent/time.rb @@ -15,6 +15,7 @@ # require 'time' +require 'timezone' require 'msgpack' module Fluent @@ -101,4 +102,96 @@ def method_missing(name, *args, &block) @sec.send(name, *args, &block) end end + + class TimeFormatter + def initialize(format, localtime, timezone = nil) + @tc1 = 0 + @tc1_str = nil + @tc2 = 0 + @tc2_str = nil + + if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/ + define_singleton_method(:format) {|time| + format_with_subsec(time) + } + else + define_singleton_method(:format) {|time| + format_without_subsec(time) + } + end + + if formatter = Fluent::Timezone.formatter(timezone, format) + define_singleton_method(:format_nocache) {|time| + formatter.call(time) + } + return + end + + if format + if localtime + define_singleton_method(:format_nocache) {|time| + Time.at(time).strftime(format) + } + else + define_singleton_method(:format_nocache) {|time| + Time.at(time).utc.strftime(format) + } + end + else + if localtime + define_singleton_method(:format_nocache) {|time| + Time.at(time).iso8601 + } + else + define_singleton_method(:format_nocache) {|time| + Time.at(time).utc.iso8601 + } + end + end + end + + def format_without_subsec(time) + if @tc1 == time + return @tc1_str + elsif @tc2 == time + return @tc2_str + else + str = format_nocache(time) + if @tc1 < @tc2 + @tc1 = time + @tc1_str = str + else + @tc2 = time + @tc2_str = str + end + return str + end + end + + def format_with_subsec(time) + if Fluent::EventTime.eq?(@tc1, time) + return @tc1_str + elsif Fluent::EventTime.eq?(@tc2, time) + return @tc2_str + else + str = format_nocache(time) + if @tc1 < @tc2 + @tc1 = time + @tc1_str = str + else + @tc2 = time + @tc2_str = str + end + return str + end + end + + def format(time) + # will be overridden in initialize + end + + def format_nocache(time) + # will be overridden in initialize + end + end end From 5c0b5443d96d9f37cd1695254c18c94fac30a21d Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 21 Jun 2016 18:12:31 +0900 Subject: [PATCH 05/21] Make code slim, and reduce warnings --- lib/fluent/compat/set_time_key_mixin.rb | 6 +-- lib/fluent/mixin.rb | 4 +- lib/fluent/time.rb | 54 +++++++------------------ 3 files changed, 19 insertions(+), 45 deletions(-) diff --git a/lib/fluent/compat/set_time_key_mixin.rb b/lib/fluent/compat/set_time_key_mixin.rb index 3df3830db9..193cbd3e3e 100644 --- a/lib/fluent/compat/set_time_key_mixin.rb +++ b/lib/fluent/compat/set_time_key_mixin.rb @@ -16,12 +16,12 @@ require 'fluent/config/error' require 'fluent/compat/record_filter_mixin' -require 'fluent/compat/time_formatter' +require 'fluent/time' +require 'fluent/timezone' module Fluent module Compat module SetTimeKeyMixin - require 'fluent/timezone' include RecordFilterMixin attr_accessor :include_time_key, :time_key, :localtime, :timezone @@ -55,7 +55,7 @@ def configure(conf) Fluent::Timezone.validate!(@timezone) end - @timef = Fluent::Compat::TimeFormatter.new(@time_format, @localtime, @timezone) + @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone) end end diff --git a/lib/fluent/mixin.rb b/lib/fluent/mixin.rb index 6d9d694d6a..652eadfca3 100644 --- a/lib/fluent/mixin.rb +++ b/lib/fluent/mixin.rb @@ -20,7 +20,7 @@ require 'fluent/compat/set_tag_key_mixin' require 'fluent/compat/type_converter' -require 'fluent/time' +require 'fluent/time' # Fluent::TimeFormatter module Fluent RecordFilterMixin = Fluent::Compat::RecordFilterMixin @@ -28,6 +28,4 @@ module Fluent SetTimeKeyMixin = Fluent::Compat::SetTimeKeyMixin SetTagKeyMixin = Fluent::Compat::SetTagKeyMixin TypeConverter = Fluent::Compat::TypeConverter - - TimeFormatter = Fluent::TimeFormatter end diff --git a/lib/fluent/time.rb b/lib/fluent/time.rb index 0852923374..df9c1c419e 100644 --- a/lib/fluent/time.rb +++ b/lib/fluent/time.rb @@ -15,8 +15,8 @@ # require 'time' -require 'timezone' require 'msgpack' +require 'fluent/timezone' module Fluent class EventTime @@ -111,43 +111,19 @@ def initialize(format, localtime, timezone = nil) @tc2_str = nil if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/ - define_singleton_method(:format) {|time| - format_with_subsec(time) - } + define_singleton_method(:format, method(:format_with_subsec)) else - define_singleton_method(:format) {|time| - format_without_subsec(time) - } + define_singleton_method(:format, method(:format_without_subsec)) end - if formatter = Fluent::Timezone.formatter(timezone, format) - define_singleton_method(:format_nocache) {|time| - formatter.call(time) - } - return - end - - if format - if localtime - define_singleton_method(:format_nocache) {|time| - Time.at(time).strftime(format) - } - else - define_singleton_method(:format_nocache) {|time| - Time.at(time).utc.strftime(format) - } - end - else - if localtime - define_singleton_method(:format_nocache) {|time| - Time.at(time).iso8601 - } - else - define_singleton_method(:format_nocache) {|time| - Time.at(time).utc.iso8601 - } - end - end + formatter = Fluent::Timezone.formatter(timezone, format) + @format_nocache = case + when formatter then formatter + when format && localtime then ->(time){ Time.at(time).strftime(format) } + when format then ->(time){ Time.at(time).utc.strftime(format) } + when localtime then ->(time){ Time.at(time).iso8601 } + else ->(time){ Time.at(time).utc.iso8601 } + end end def format_without_subsec(time) @@ -186,12 +162,12 @@ def format_with_subsec(time) end end - def format(time) - # will be overridden in initialize - end + ## Dynamically defined in #initialize + # def format(time) + # end def format_nocache(time) - # will be overridden in initialize + @format_nocache.call(time) end end end From a4b55fa5e9d00582e627b776a32a2b516dcde0ad Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 21 Jun 2016 18:13:36 +0900 Subject: [PATCH 06/21] separate tests of time formatter: it is not a Formatter plugin --- test/test_formatter.rb | 178 ---------------------------------- test/test_time_formatter.rb | 186 ++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 178 deletions(-) create mode 100644 test/test_time_formatter.rb diff --git a/test/test_formatter.rb b/test/test_formatter.rb index 54540ac2f9..235a33b17f 100644 --- a/test/test_formatter.rb +++ b/test/test_formatter.rb @@ -430,184 +430,6 @@ def test_find_formatter(data) end end - class TimeFormatterTest < ::Test::Unit::TestCase - include FormatterTest - - def setup - @time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i - @fmt = "%Y%m%d %H%M%z" # YYYYMMDD HHMM[+-]HHMM - end - - def format(format, localtime, timezone) - formatter = Fluent::TimeFormatter.new(format, localtime, timezone) - formatter.format(@time) - end - - def test_default_utc_nil - assert_equal("2014-09-27T00:00:00Z", format(nil, false, nil)) - end - - def test_default_utc_pHH_MM - assert_equal("2014-09-27T01:30:00+01:30", format(nil, false, "+01:30")) - end - - def test_default_utc_nHH_MM - assert_equal("2014-09-26T22:30:00-01:30", format(nil, false, "-01:30")) - end - - def test_default_utc_pHHMM - assert_equal("2014-09-27T02:30:00+02:30", format(nil, false, "+0230")) - end - - def test_default_utc_nHHMM - assert_equal("2014-09-26T21:30:00-02:30", format(nil, false, "-0230")) - end - - def test_default_utc_pHH - assert_equal("2014-09-27T03:00:00+03:00", format(nil, false, "+03")) - end - - def test_default_utc_nHH - assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "-03")) - end - - def test_default_utc_timezone_1 - # Asia/Tokyo (+09:00) does not have daylight saving time. - assert_equal("2014-09-27T09:00:00+09:00", format(nil, false, "Asia/Tokyo")) - end - - def test_default_utc_timezone_2 - # Pacific/Honolulu (-10:00) does not have daylight saving time. - assert_equal("2014-09-26T14:00:00-10:00", format(nil, false, "Pacific/Honolulu")) - end - - def test_default_utc_timezone_3 - # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time. - assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "America/Argentina/Buenos_Aires")) - end - - def test_default_utc_timezone_4 - # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its - # UTC offset in DST is +02:00. In September, Europe/Paris is in DST. - assert_equal("2014-09-27T02:00:00+02:00", format(nil, false, "Europe/Paris")) - end - - def test_default_utc_timezone_5 - # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its - # UTC offset in DST is +02:00. In January, Europe/Paris is not in DST. - @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i - assert_equal("2014-01-24T01:00:00+01:00", format(nil, false, "Europe/Paris")) - end - - def test_default_utc_invalid - assert_equal("2014-09-27T00:00:00Z", format(nil, false, "Invalid")) - end - - def test_default_localtime_nil_1 - with_timezone("UTC-04") do - assert_equal("2014-09-27T04:00:00+04:00", format(nil, true, nil)) - end - end - - def test_default_localtime_nil_2 - with_timezone("UTC+05") do - assert_equal("2014-09-26T19:00:00-05:00", format(nil, true, nil)) - end - end - - def test_default_localtime_timezone - # 'timezone' takes precedence over 'localtime'. - with_timezone("UTC-06") do - assert_equal("2014-09-27T07:00:00+07:00", format(nil, true, "+07")) - end - end - - def test_specific_utc_nil - assert_equal("20140927 0000+0000", format(@fmt, false, nil)) - end - - def test_specific_utc_pHH_MM - assert_equal("20140927 0830+0830", format(@fmt, false, "+08:30")) - end - - def test_specific_utc_nHH_MM - assert_equal("20140926 1430-0930", format(@fmt, false, "-09:30")) - end - - def test_specific_utc_pHHMM - assert_equal("20140927 1030+1030", format(@fmt, false, "+1030")) - end - - def test_specific_utc_nHHMM - assert_equal("20140926 1230-1130", format(@fmt, false, "-1130")) - end - - def test_specific_utc_pHH - assert_equal("20140927 1200+1200", format(@fmt, false, "+12")) - end - - def test_specific_utc_nHH - assert_equal("20140926 1100-1300", format(@fmt, false, "-13")) - end - - def test_specific_utc_timezone_1 - # Europe/Moscow (+04:00) does not have daylight saving time. - assert_equal("20140927 0400+0400", format(@fmt, false, "Europe/Moscow")) - end - - def test_specific_utc_timezone_2 - # Pacific/Galapagos (-06:00) does not have daylight saving time. - assert_equal("20140926 1800-0600", format(@fmt, false, "Pacific/Galapagos")) - end - - def test_specific_utc_timezone_3 - # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time. - assert_equal("20140926 2100-0300", format(@fmt, false, "America/Argentina/Buenos_Aires")) - end - - def test_specific_utc_timezone_4 - # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its - # UTC offset in DST is -07:00. In September, America/Los_Angeles is in DST. - assert_equal("20140926 1700-0700", format(@fmt, false, "America/Los_Angeles")) - end - - def test_specific_utc_timezone_5 - # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its - # UTC offset in DST is -07:00. In January, America/Los_Angeles is not in DST. - @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i - assert_equal("20140123 1600-0800", format(@fmt, false, "America/Los_Angeles")) - end - - def test_specific_utc_invalid - assert_equal("20140927 0000+0000", format(@fmt, false, "Invalid")) - end - - def test_specific_localtime_nil_1 - with_timezone("UTC-07") do - assert_equal("20140927 0700+0700", format(@fmt, true, nil)) - end - end - - def test_specific_localtime_nil_2 - with_timezone("UTC+08") do - assert_equal("20140926 1600-0800", format(@fmt, true, nil)) - end - end - - def test_specific_localtime_timezone - # 'timezone' takes precedence over 'localtime'. - with_timezone("UTC-09") do - assert_equal("20140926 1400-1000", format(@fmt, true, "-10")) - end - end - - def test_format_with_subsec - time = Fluent::EventTime.new(@time) - formatter = Fluent::TimeFormatter.new("%Y%m%d %H%M.%N", false, nil) - assert_equal("20140927 0000.000000000", formatter.format(time)) - end - end - class TimeConfigTest < ::Test::Unit::TestCase include FormatterTest diff --git a/test/test_time_formatter.rb b/test/test_time_formatter.rb new file mode 100644 index 0000000000..7a2248bec9 --- /dev/null +++ b/test/test_time_formatter.rb @@ -0,0 +1,186 @@ +require_relative 'helper' +require 'fluent/test' +require 'fluent/time' + +class TimeFormatterTest < ::Test::Unit::TestCase + def with_timezone(tz) + oldtz, ENV['TZ'] = ENV['TZ'], tz + yield + ensure + ENV['TZ'] = oldtz + end + + def setup + @time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i + @fmt = "%Y%m%d %H%M%z" # YYYYMMDD HHMM[+-]HHMM + end + + def format(format, localtime, timezone) + formatter = Fluent::TimeFormatter.new(format, localtime, timezone) + formatter.format(@time) + end + + def test_default_utc_nil + assert_equal("2014-09-27T00:00:00Z", format(nil, false, nil)) + end + + def test_default_utc_pHH_MM + assert_equal("2014-09-27T01:30:00+01:30", format(nil, false, "+01:30")) + end + + def test_default_utc_nHH_MM + assert_equal("2014-09-26T22:30:00-01:30", format(nil, false, "-01:30")) + end + + def test_default_utc_pHHMM + assert_equal("2014-09-27T02:30:00+02:30", format(nil, false, "+0230")) + end + + def test_default_utc_nHHMM + assert_equal("2014-09-26T21:30:00-02:30", format(nil, false, "-0230")) + end + + def test_default_utc_pHH + assert_equal("2014-09-27T03:00:00+03:00", format(nil, false, "+03")) + end + + def test_default_utc_nHH + assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "-03")) + end + + def test_default_utc_timezone_1 + # Asia/Tokyo (+09:00) does not have daylight saving time. + assert_equal("2014-09-27T09:00:00+09:00", format(nil, false, "Asia/Tokyo")) + end + + def test_default_utc_timezone_2 + # Pacific/Honolulu (-10:00) does not have daylight saving time. + assert_equal("2014-09-26T14:00:00-10:00", format(nil, false, "Pacific/Honolulu")) + end + + def test_default_utc_timezone_3 + # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time. + assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "America/Argentina/Buenos_Aires")) + end + + def test_default_utc_timezone_4 + # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its + # UTC offset in DST is +02:00. In September, Europe/Paris is in DST. + assert_equal("2014-09-27T02:00:00+02:00", format(nil, false, "Europe/Paris")) + end + + def test_default_utc_timezone_5 + # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its + # UTC offset in DST is +02:00. In January, Europe/Paris is not in DST. + @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i + assert_equal("2014-01-24T01:00:00+01:00", format(nil, false, "Europe/Paris")) + end + + def test_default_utc_invalid + assert_equal("2014-09-27T00:00:00Z", format(nil, false, "Invalid")) + end + + def test_default_localtime_nil_1 + with_timezone("UTC-04") do + assert_equal("2014-09-27T04:00:00+04:00", format(nil, true, nil)) + end + end + + def test_default_localtime_nil_2 + with_timezone("UTC+05") do + assert_equal("2014-09-26T19:00:00-05:00", format(nil, true, nil)) + end + end + + def test_default_localtime_timezone + # 'timezone' takes precedence over 'localtime'. + with_timezone("UTC-06") do + assert_equal("2014-09-27T07:00:00+07:00", format(nil, true, "+07")) + end + end + + def test_specific_utc_nil + assert_equal("20140927 0000+0000", format(@fmt, false, nil)) + end + + def test_specific_utc_pHH_MM + assert_equal("20140927 0830+0830", format(@fmt, false, "+08:30")) + end + + def test_specific_utc_nHH_MM + assert_equal("20140926 1430-0930", format(@fmt, false, "-09:30")) + end + + def test_specific_utc_pHHMM + assert_equal("20140927 1030+1030", format(@fmt, false, "+1030")) + end + + def test_specific_utc_nHHMM + assert_equal("20140926 1230-1130", format(@fmt, false, "-1130")) + end + + def test_specific_utc_pHH + assert_equal("20140927 1200+1200", format(@fmt, false, "+12")) + end + + def test_specific_utc_nHH + assert_equal("20140926 1100-1300", format(@fmt, false, "-13")) + end + + def test_specific_utc_timezone_1 + # Europe/Moscow (+04:00) does not have daylight saving time. + assert_equal("20140927 0400+0400", format(@fmt, false, "Europe/Moscow")) + end + + def test_specific_utc_timezone_2 + # Pacific/Galapagos (-06:00) does not have daylight saving time. + assert_equal("20140926 1800-0600", format(@fmt, false, "Pacific/Galapagos")) + end + + def test_specific_utc_timezone_3 + # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time. + assert_equal("20140926 2100-0300", format(@fmt, false, "America/Argentina/Buenos_Aires")) + end + + def test_specific_utc_timezone_4 + # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its + # UTC offset in DST is -07:00. In September, America/Los_Angeles is in DST. + assert_equal("20140926 1700-0700", format(@fmt, false, "America/Los_Angeles")) + end + + def test_specific_utc_timezone_5 + # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its + # UTC offset in DST is -07:00. In January, America/Los_Angeles is not in DST. + @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i + assert_equal("20140123 1600-0800", format(@fmt, false, "America/Los_Angeles")) + end + + def test_specific_utc_invalid + assert_equal("20140927 0000+0000", format(@fmt, false, "Invalid")) + end + + def test_specific_localtime_nil_1 + with_timezone("UTC-07") do + assert_equal("20140927 0700+0700", format(@fmt, true, nil)) + end + end + + def test_specific_localtime_nil_2 + with_timezone("UTC+08") do + assert_equal("20140926 1600-0800", format(@fmt, true, nil)) + end + end + + def test_specific_localtime_timezone + # 'timezone' takes precedence over 'localtime'. + with_timezone("UTC-09") do + assert_equal("20140926 1400-1000", format(@fmt, true, "-10")) + end + end + + def test_format_with_subsec + time = Fluent::EventTime.new(@time) + formatter = Fluent::TimeFormatter.new("%Y%m%d %H%M.%N", false, nil) + assert_equal("20140927 0000.000000000", formatter.format(time)) + end +end From e44aced4455fae67ccfb919b9e8853bd5b9d961a Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 21 Jun 2016 18:14:11 +0900 Subject: [PATCH 07/21] add Inject plugin helper instead of SetTimeKeyMixin/SetTagKeyMixin --- lib/fluent/plugin_helper/inject.rb | 92 ++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 lib/fluent/plugin_helper/inject.rb diff --git a/lib/fluent/plugin_helper/inject.rb b/lib/fluent/plugin_helper/inject.rb new file mode 100644 index 0000000000..7618c14b28 --- /dev/null +++ b/lib/fluent/plugin_helper/inject.rb @@ -0,0 +1,92 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/event' +require 'fluent/plugin_helper/formatter' + +module Fluent + module PluginHelper + module Inject + include Fluent::PluginHelper::Formatter + + def inject_record(tag, time, record) + return record unless @_inject_tag_key || @_inject_time_key + + r = record.dup + if @_inject_tag_key + r[@_inject_tag_key] = tag + end + if @_inject_time_key + r[@_inject_time_key] = @_inject_time_formatter.call(time) + end + end + + def inject_event_stream(tag, es) + return es unless @_inject_tag_key || @_inject_time_key + + new_es = Fluent::MultiEventStream.new + es.each do |time, record| + r = record.dup + if @_inject_tag_key + r[@_inject_tag_key] = tag + end + if @_inject_time_key + r[@_inject_time_key] = @_inject_time_formatter.call(time) + end + new_es.add(time, r) + end + + new_es + end + + def self.included(mod) + mod.instance_eval do + config_section :inject, required: false, multi: false, param_name: :inject_config do + config_param :tag_key, :string, default: nil + config_param :time_key, :string, default: nil + config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float + config_param :time_format, :string, default: nil + config_param :timezone, :string, default: nil # localtime + end + end + end + + def initialize + super + @_inject_tag_key = nil + @_inject_time_key = nil + @_inject_time_formatter = nil + end + + def configure(conf) + super + + if @inject_config + @_inject_tag_key = @inject_config.tag_key + @_inject_time_key = @inject_config.time_key + if @_inject_time_key + @_inject_time_formatter = case @inject_config.time_type + when :float then ->(time){ time.to_r.to_f } + when :unixtime then ->(time){ time.to_i } + else + Fluent::TimeFormatter.new(@inject_config.time_format, @inject_config.timezone) + end + end + end + end + end + end +end From 2047ffa35fcd190102be6dd1b3cb2a016643ea56 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 22 Jun 2016 16:43:56 +0900 Subject: [PATCH 08/21] fix 3 space indentation to 2 --- lib/fluent/compat/set_time_key_mixin.rb | 70 ++++++++++++------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/lib/fluent/compat/set_time_key_mixin.rb b/lib/fluent/compat/set_time_key_mixin.rb index 193cbd3e3e..7abbb6398e 100644 --- a/lib/fluent/compat/set_time_key_mixin.rb +++ b/lib/fluent/compat/set_time_key_mixin.rb @@ -21,49 +21,49 @@ module Fluent module Compat - module SetTimeKeyMixin - include RecordFilterMixin + module SetTimeKeyMixin + include RecordFilterMixin - attr_accessor :include_time_key, :time_key, :localtime, :timezone + attr_accessor :include_time_key, :time_key, :localtime, :timezone - def configure(conf) - @include_time_key = false - @localtime = false - @timezone = nil + def configure(conf) + @include_time_key = false + @localtime = false + @timezone = nil - super + super - if s = conf['include_time_key'] - include_time_key = Fluent::Config.bool_value(s) - raise Fluent::ConfigError, "Invalid boolean expression '#{s}' for include_time_key parameter" if include_time_key.nil? + if s = conf['include_time_key'] + include_time_key = Fluent::Config.bool_value(s) + raise Fluent::ConfigError, "Invalid boolean expression '#{s}' for include_time_key parameter" if include_time_key.nil? - @include_time_key = include_time_key - end + @include_time_key = include_time_key + end - if @include_time_key - @time_key = conf['time_key'] || 'time' - @time_format = conf['time_format'] + if @include_time_key + @time_key = conf['time_key'] || 'time' + @time_format = conf['time_format'] - if conf['localtime'] - @localtime = true - elsif conf['utc'] - @localtime = false - end + if conf['localtime'] + @localtime = true + elsif conf['utc'] + @localtime = false + end - if conf['timezone'] - @timezone = conf['timezone'] - Fluent::Timezone.validate!(@timezone) - end + if conf['timezone'] + @timezone = conf['timezone'] + Fluent::Timezone.validate!(@timezone) + end - @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone) - end - end + @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone) + end + end - def filter_record(tag, time, record) - super + def filter_record(tag, time, record) + super - record[@time_key] = @timef.format(time) if @include_time_key - end - end - end - end + record[@time_key] = @timef.format(time) if @include_time_key + end + end + end +end From f2be0a79c26d39ebfef9704899ea455b63ffc934 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 22 Jun 2016 16:51:59 +0900 Subject: [PATCH 09/21] add #call to use TimeFormatter in same way with proc --- lib/fluent/time.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/fluent/time.rb b/lib/fluent/time.rb index df9c1c419e..c8d95d1dfc 100644 --- a/lib/fluent/time.rb +++ b/lib/fluent/time.rb @@ -112,8 +112,10 @@ def initialize(format, localtime, timezone = nil) if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/ define_singleton_method(:format, method(:format_with_subsec)) + define_singleton_method(:call, method(:format_with_subsec)) else define_singleton_method(:format, method(:format_without_subsec)) + define_singleton_method(:call, method(:format_with_subsec)) end formatter = Fluent::Timezone.formatter(timezone, format) From d3aa0e4607d087f6ad8d0b81d40c43c3133e2d28 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 22 Jun 2016 16:52:42 +0900 Subject: [PATCH 10/21] add hostname injection --- lib/fluent/plugin_helper/inject.rb | 34 ++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/fluent/plugin_helper/inject.rb b/lib/fluent/plugin_helper/inject.rb index 7618c14b28..e4a6c3c81f 100644 --- a/lib/fluent/plugin_helper/inject.rb +++ b/lib/fluent/plugin_helper/inject.rb @@ -15,31 +15,37 @@ # require 'fluent/event' -require 'fluent/plugin_helper/formatter' +require 'time' module Fluent module PluginHelper module Inject - include Fluent::PluginHelper::Formatter - def inject_record(tag, time, record) - return record unless @_inject_tag_key || @_inject_time_key + return record unless @_inject_enabled r = record.dup + if @_inject_hostname_key + r[@_inject_hostname_key] = @_inject_hostname + end if @_inject_tag_key r[@_inject_tag_key] = tag end if @_inject_time_key r[@_inject_time_key] = @_inject_time_formatter.call(time) end + + r end def inject_event_stream(tag, es) - return es unless @_inject_tag_key || @_inject_time_key + return es unless @_inject_enabled new_es = Fluent::MultiEventStream.new es.each do |time, record| r = record.dup + if @_inject_hostname_key + r[@_inject_hostname_key] = @_inject_hostname + end if @_inject_tag_key r[@_inject_tag_key] = tag end @@ -55,17 +61,21 @@ def inject_event_stream(tag, es) def self.included(mod) mod.instance_eval do config_section :inject, required: false, multi: false, param_name: :inject_config do + config_param :hostname_key, :string, default: nil + config_param :hostname, :string, default: nil config_param :tag_key, :string, default: nil config_param :time_key, :string, default: nil config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float config_param :time_format, :string, default: nil - config_param :timezone, :string, default: nil # localtime + config_param :timezone, :string, default: "#{Time.now.strftime('%z')}" # localtime end end end def initialize super + @_inject_hostname_key = nil + @_inject_hostname = nil @_inject_tag_key = nil @_inject_time_key = nil @_inject_time_formatter = nil @@ -75,6 +85,14 @@ def configure(conf) super if @inject_config + @_inject_hostname_key = @inject_config.hostname_key + if @_inject_hostname_key + @_inject_hostname = @inject_config.hostname + unless @_inject_hostname + @_inject_hostname = Socket.gethostname + log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname + end + end @_inject_tag_key = @inject_config.tag_key @_inject_time_key = @inject_config.time_key if @_inject_time_key @@ -82,9 +100,11 @@ def configure(conf) when :float then ->(time){ time.to_r.to_f } when :unixtime then ->(time){ time.to_i } else - Fluent::TimeFormatter.new(@inject_config.time_format, @inject_config.timezone) + Fluent::TimeFormatter.new(@inject_config.time_format, false, @inject_config.timezone) end end + + @_inject_enabled = @_inject_hostname_key || @_inject_tag_key || @_inject_time_key end end end From 46ebbd30f5227cbd7f0279d9dbd7bc21e2b9a13f Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 22 Jun 2016 16:52:55 +0900 Subject: [PATCH 11/21] add tests for inject plugin helper --- test/plugin_helper/test_inject.rb | 385 ++++++++++++++++++++++++++++++ 1 file changed, 385 insertions(+) create mode 100644 test/plugin_helper/test_inject.rb diff --git a/test/plugin_helper/test_inject.rb b/test/plugin_helper/test_inject.rb new file mode 100644 index 0000000000..342842bd17 --- /dev/null +++ b/test/plugin_helper/test_inject.rb @@ -0,0 +1,385 @@ +require_relative '../helper' +require 'fluent/plugin_helper/inject' +require 'fluent/event' +require 'time' + +class InjectHelperTest < Test::Unit::TestCase + class Dummy < Fluent::Plugin::TestBase + helpers :inject + end + + def config_inject_section(hash = {}) + config_element('ROOT', '', {}, [config_element('inject', '', hash)]) + end + + setup do + @d = Dummy.new + end + + teardown do + if @d + @d.stop unless @d.stopped? + @d.shutdown unless @d.shutdown? + @d.close unless @d.closed? + @d.terminate unless @d.terminated? + end + end + + test 'do nothing in default' do + @d.configure(config_inject_section()) + @d.start + assert_nil @d.instance_eval{ @_inject_host_key } + assert_nil @d.instance_eval{ @_inject_host_name } + assert_nil @d.instance_eval{ @_inject_tag_key } + assert_nil @d.instance_eval{ @_inject_time_key } + assert_nil @d.instance_eval{ @_inject_time_formatter } + + time = event_time() + record = {"key1" => "value1", "key2" => 2} + assert_equal record, @d.inject_record('tag', time, record) + assert_equal record.object_id, @d.inject_record('tag', time, record).object_id + + es0 = Fluent::OneEventStream.new(time, {"key1" => "v", "key2" => 0}) + + es1 = Fluent::ArrayEventStream.new([ [time, {"key1" => "a", "key2" => 1}], [time, {"key1" => "b", "key2" => 2}] ]) + + es2 = Fluent::MultiEventStream.new + es2.add(event_time(), {"key1" => "a", "key2" => 1}) + es2.add(event_time(), {"key1" => "b", "key2" => 2}) + + es3 = Fluent::MessagePackEventStream.new(es2.to_msgpack_stream) + + [es0, es1, es2, es3].each do |es| + assert_equal es, @d.inject_event_stream('tag', es), "failed for #{es.class}" + assert_equal es.object_id, @d.inject_event_stream('tag', es).object_id, "failed for #{es.class}" + end + end + + test 'can be configured as specified' do + @d.configure(config_inject_section( + "hostname_key" => "hostname", + "hostname" => "myhost.local", + "tag_key" => "tag", + "time_key" => "time", + "time_type" => "string", + "time_format" => "%Y-%m-%d %H:%M:%S.%N", + "timezone" => "-0700", + )) + + assert_equal "hostname", @d.instance_eval{ @_inject_hostname_key } + assert_equal "myhost.local", @d.instance_eval{ @_inject_hostname } + assert_equal "tag", @d.instance_eval{ @_inject_tag_key } + assert_equal "time", @d.instance_eval{ @_inject_time_key } + assert_equal :string, @d.instance_eval{ @inject_config.time_type } + assert_not_nil @d.instance_eval{ @_inject_time_formatter } + end + + sub_test_case 'using inject_record' do + test 'injects hostname automatically detected' do + detected_hostname = `hostname`.chomp + @d.configure(config_inject_section("hostname_key" => "host")) + logs = @d.log.out.logs + assert{ logs.first.include?('[info]: using hostname for specified field host_key="host" host_name="SATOSHI-no-MacBook-Air.local"') } + @d.start + + time = event_time() + record = {"key1" => "value1", "key2" => 2} + assert_equal record.merge({"host" => detected_hostname}), @d.inject_record('tag', time, record) + end + + test 'injects hostname as specified value' do + @d.configure(config_inject_section("hostname_key" => "host", "hostname" => "myhost.yay.local")) + @d.start + + time = event_time() + record = {"key1" => "value1", "key2" => 2} + assert_equal record.merge({"host" => "myhost.yay.local"}), @d.inject_record('tag', time, record) + end + + test 'injects tag into specified key' do + @d.configure(config_inject_section("tag_key" => "mytag")) + @d.start + + time = event_time() + record = {"key1" => "value1", "key2" => 2} + assert_equal record.merge({"mytag" => "tag.test"}), @d.inject_record('tag.test', time, record) + end + + test 'injects time as floating point value into specified key as default' do + time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i # 1466464211 in unix time + time_subsecond = 320_101_224 + time = Fluent::EventTime.new(time_in_unix, time_subsecond) + float_time = 1466464211.320101 # microsecond precision in float + + @d.configure(config_inject_section("time_key" => "timedata")) + @d.start + + record = {"key1" => "value1", "key2" => 2} + assert_equal record.merge({"timedata" => float_time}), @d.inject_record('tag', time, record) + end + + test 'injects time as unix time into specified key' do + time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i + time_subsecond = 320_101_224 + time = Fluent::EventTime.new(time_in_unix, time_subsecond) + int_time = 1466464211 + + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime")) + @d.start + + record = {"key1" => "value1", "key2" => 2} + assert_equal record.merge({"timedata" => int_time}), @d.inject_record('tag', time, record) + end + + test 'injects time as formatted string in localtime if timezone not specified' do + local_timezone = Time.now.strftime('%z') + time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i + time_subsecond = 320_101_224 + time = Fluent::EventTime.new(time_in_unix, time_subsecond) + + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z")) + @d.start + + record = {"key1" => "value1", "key2" => 2} + assert_equal record.merge({"timedata" => "2016_06_21 08:10:11 #{local_timezone}"}), @d.inject_record('tag', time, record) + end + + test 'injects time as formatted string with nanosecond in localtime if timezone not specified' do + local_timezone = Time.now.strftime('%z') + time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i + time_subsecond = 320_101_224 + time = Fluent::EventTime.new(time_in_unix, time_subsecond) + + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%N %z")) + @d.start + + record = {"key1" => "value1", "key2" => 2} + assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320101224 #{local_timezone}"}), @d.inject_record('tag', time, record) + end + + test 'injects time as formatted string with millisecond in localtime if timezone not specified' do + local_timezone = Time.now.strftime('%z') + time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i + time_subsecond = 320_101_224 + time = Fluent::EventTime.new(time_in_unix, time_subsecond) + + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%3N %z")) + @d.start + + record = {"key1" => "value1", "key2" => 2} + assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320 #{local_timezone}"}), @d.inject_record('tag', time, record) + end + + test 'injects time as formatted string in specified timezone' do + time_in_unix = Time.parse("2016-06-21 08:10:11 +0000").to_i + time_subsecond = 320_101_224 + time = Fluent::EventTime.new(time_in_unix, time_subsecond) + + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z", "timezone" => "-0800")) + @d.start + + record = {"key1" => "value1", "key2" => 2} + assert_equal record.merge({"timedata" => "2016_06_21 00:10:11 -0800"}), @d.inject_record('tag', time, record) + end + + test 'injects hostname, tag and time' do + time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i + time_subsecond = 320_101_224 + time = Fluent::EventTime.new(time_in_unix, time_subsecond) + + @d.configure(config_inject_section( + "hostname_key" => "hostnamedata", + "hostname" => "myname.local", + "tag_key" => "tagdata", + "time_key" => "timedata", + "time_type" => "string", + "time_format" => "%Y_%m_%d %H:%M:%S.%N %z", + "timezone" => "+0000", + )) + @d.start + + record = {"key1" => "value1", "key2" => 2} + injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => "2016_06_20 23:10:11.320101224 +0000"} + assert_equal record.merge(injected), @d.inject_record('tag', time, record) + end + end + sub_test_case 'using inject_event_stream' do + local_timezone = Time.now.strftime('%z') + time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i + time_subsecond = 320_101_224 + time = Fluent::EventTime.new(time_in_unix, time_subsecond) + int_time = 1466464211 + float_time = 1466464211.320101 + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects hostname automatically detected' do |data| + detected_hostname = `hostname`.chomp + @d.configure(config_inject_section("hostname_key" => "host")) + logs = @d.log.out.logs + assert{ logs.first.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } + @d.start + + injected = {"host" => detected_hostname} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects hostname as specified value' do |data| + @d.configure(config_inject_section("hostname_key" => "host", "hostname" => "myhost.yay.local")) + @d.start + + injected = {"host" => "myhost.yay.local"} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects tag into specified key' do |data| + @d.configure(config_inject_section("tag_key" => "mytag")) + @d.start + + injected = {"mytag" => "tag"} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects time as floating point value into specified key as default' do |data| + @d.configure(config_inject_section("time_key" => "timedata")) + @d.start + + injected = {"timedata" => float_time} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects time as unix time into specified key' do |data| + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime")) + @d.start + + injected = {"timedata" => int_time} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects time as formatted string in localtime if timezone not specified' do |data| + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z")) + @d.start + + injected = {"timedata" => "2016_06_21 08:10:11 #{local_timezone}"} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects time as formatted string with nanosecond in localtime if timezone not specified' do |data| + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%N %z")) + @d.start + + injected = {"timedata" => "2016_06_21 08:10:11.320101224 #{local_timezone}"} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects time as formatted string with millisecond in localtime if timezone not specified' do |data| + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%3N %z")) + @d.start + + injected = {"timedata" => "2016_06_21 08:10:11.320 #{local_timezone}"} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects time as formatted string in specified timezone' do |data| + @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z", "timezone" => "-0800")) + @d.start + + injected = {"timedata" => Time.at(int_time).localtime("-08:00").strftime("%Y_%m_%d %H:%M:%S -0800")} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + + data( + "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), + "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]), + ) + test 'injects hostname, tag and time' do |data| + @d.configure(config_inject_section( + "hostname_key" => "hostnamedata", + "hostname" => "myname.local", + "tag_key" => "tagdata", + "time_key" => "timedata", + "time_type" => "string", + "time_format" => "%Y_%m_%d %H:%M:%S.%N %z", + "timezone" => "+0000", + )) + @d.start + + injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => "2016_06_20 23:10:11.320101224 +0000"} + expected_es = Fluent::MultiEventStream.new + data.each do |t, r| + expected_es.add(t, r.merge(injected)) + end + assert_equal expected_es, @d.inject_event_stream('tag', data) + end + end +end From bd9b9599dc8fe4ec0483a5896df9142395f1d8e6 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 28 Jun 2016 14:21:04 +0900 Subject: [PATCH 12/21] initialize global logger everytime to avoid reusing hacked logger by other tests --- lib/fluent/test.rb | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/fluent/test.rb b/lib/fluent/test.rb index b60e771642..4ad6507e6c 100644 --- a/lib/fluent/test.rb +++ b/lib/fluent/test.rb @@ -25,15 +25,19 @@ require 'fluent/test/formatter_test' require 'serverengine' -dl_opts = {} -dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO -logdev = Fluent::Test::DummyLogDevice.new -logger = ServerEngine::DaemonLogger.new(logdev, dl_opts) -$log ||= Fluent::Log.new(logger) module Fluent module Test + def self.dummy_logger + dl_opts = {log_level: ServerEngine::DaemonLogger::INFO} + logdev = Fluent::Test::DummyLogDevice.new + logger = ServerEngine::DaemonLogger.new(logdev, dl_opts) + Fluent::Log.new(logger) + end + def self.setup + $log = dummy_logger + Fluent.__send__(:remove_const, :Engine) engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new) @@ -49,3 +53,4 @@ def self.setup end end +$log ||= Fluent::Test.dummy_logger From 08124484a59c8145eb41792f55dde2bf9557c8f3 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 28 Jun 2016 14:21:16 +0900 Subject: [PATCH 13/21] code style --- test/test_event.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_event.rb b/test/test_event.rb index fc5f301d8d..3543bcc3b1 100644 --- a/test/test_event.rb +++ b/test/test_event.rb @@ -50,8 +50,8 @@ def setup end test 'slice' do - assert_equal 0, @es.slice(1,1).size - assert_equal 0, @es.slice(0,0).size + assert_equal 0, @es.slice(1, 1).size + assert_equal 0, @es.slice(0, 0).size sliced = @es.slice(0, 1) assert_kind_of EventStream, sliced From c1089bee802233e9def7bd473954284d1ef50770 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 28 Jun 2016 14:21:26 +0900 Subject: [PATCH 14/21] fix buggy tests --- test/plugin_helper/test_inject.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/plugin_helper/test_inject.rb b/test/plugin_helper/test_inject.rb index 342842bd17..e0f96711db 100644 --- a/test/plugin_helper/test_inject.rb +++ b/test/plugin_helper/test_inject.rb @@ -28,8 +28,8 @@ def config_inject_section(hash = {}) test 'do nothing in default' do @d.configure(config_inject_section()) @d.start - assert_nil @d.instance_eval{ @_inject_host_key } - assert_nil @d.instance_eval{ @_inject_host_name } + assert_nil @d.instance_eval{ @_inject_hostname_key } + assert_nil @d.instance_eval{ @_inject_hostname } assert_nil @d.instance_eval{ @_inject_tag_key } assert_nil @d.instance_eval{ @_inject_time_key } assert_nil @d.instance_eval{ @_inject_time_formatter } @@ -79,7 +79,7 @@ def config_inject_section(hash = {}) detected_hostname = `hostname`.chomp @d.configure(config_inject_section("hostname_key" => "host")) logs = @d.log.out.logs - assert{ logs.first.include?('[info]: using hostname for specified field host_key="host" host_name="SATOSHI-no-MacBook-Air.local"') } + assert{ logs.first && logs.first.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } @d.start time = event_time() @@ -219,7 +219,7 @@ def config_inject_section(hash = {}) detected_hostname = `hostname`.chomp @d.configure(config_inject_section("hostname_key" => "host")) logs = @d.log.out.logs - assert{ logs.first.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } + assert{ logs.first && logs.first.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } @d.start injected = {"host" => detected_hostname} From 95a6552befccc2d6d9825abf6546720db9aec2e7 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 28 Jun 2016 14:25:43 +0900 Subject: [PATCH 15/21] fix to add method for assert_equal --- lib/fluent/event.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/fluent/event.rb b/lib/fluent/event.rb index 4223b84fce..e68428d830 100644 --- a/lib/fluent/event.rb +++ b/lib/fluent/event.rb @@ -35,6 +35,11 @@ def empty? size == 0 end + # for tests + def ==(other) + other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream + end + def repeatable? false end From c84a5060412e38a4afa0d5685ee83e8e597e43e3 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 28 Jun 2016 14:26:14 +0900 Subject: [PATCH 16/21] rename inject helper method for easy to understand --- lib/fluent/plugin_helper/inject.rb | 4 +-- test/plugin_helper/test_inject.rb | 48 +++++++++++++++--------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/fluent/plugin_helper/inject.rb b/lib/fluent/plugin_helper/inject.rb index e4a6c3c81f..05ab9bc2f6 100644 --- a/lib/fluent/plugin_helper/inject.rb +++ b/lib/fluent/plugin_helper/inject.rb @@ -20,7 +20,7 @@ module Fluent module PluginHelper module Inject - def inject_record(tag, time, record) + def inject_values_to_record(tag, time, record) return record unless @_inject_enabled r = record.dup @@ -37,7 +37,7 @@ def inject_record(tag, time, record) r end - def inject_event_stream(tag, es) + def inject_values_to_event_stream(tag, es) return es unless @_inject_enabled new_es = Fluent::MultiEventStream.new diff --git a/test/plugin_helper/test_inject.rb b/test/plugin_helper/test_inject.rb index e0f96711db..36e993b90b 100644 --- a/test/plugin_helper/test_inject.rb +++ b/test/plugin_helper/test_inject.rb @@ -36,8 +36,8 @@ def config_inject_section(hash = {}) time = event_time() record = {"key1" => "value1", "key2" => 2} - assert_equal record, @d.inject_record('tag', time, record) - assert_equal record.object_id, @d.inject_record('tag', time, record).object_id + assert_equal record, @d.inject_values_to_record('tag', time, record) + assert_equal record.object_id, @d.inject_values_to_record('tag', time, record).object_id es0 = Fluent::OneEventStream.new(time, {"key1" => "v", "key2" => 0}) @@ -50,8 +50,8 @@ def config_inject_section(hash = {}) es3 = Fluent::MessagePackEventStream.new(es2.to_msgpack_stream) [es0, es1, es2, es3].each do |es| - assert_equal es, @d.inject_event_stream('tag', es), "failed for #{es.class}" - assert_equal es.object_id, @d.inject_event_stream('tag', es).object_id, "failed for #{es.class}" + assert_equal es, @d.inject_values_to_event_stream('tag', es), "failed for #{es.class}" + assert_equal es.object_id, @d.inject_values_to_event_stream('tag', es).object_id, "failed for #{es.class}" end end @@ -84,7 +84,7 @@ def config_inject_section(hash = {}) time = event_time() record = {"key1" => "value1", "key2" => 2} - assert_equal record.merge({"host" => detected_hostname}), @d.inject_record('tag', time, record) + assert_equal record.merge({"host" => detected_hostname}), @d.inject_values_to_record('tag', time, record) end test 'injects hostname as specified value' do @@ -93,7 +93,7 @@ def config_inject_section(hash = {}) time = event_time() record = {"key1" => "value1", "key2" => 2} - assert_equal record.merge({"host" => "myhost.yay.local"}), @d.inject_record('tag', time, record) + assert_equal record.merge({"host" => "myhost.yay.local"}), @d.inject_values_to_record('tag', time, record) end test 'injects tag into specified key' do @@ -102,7 +102,7 @@ def config_inject_section(hash = {}) time = event_time() record = {"key1" => "value1", "key2" => 2} - assert_equal record.merge({"mytag" => "tag.test"}), @d.inject_record('tag.test', time, record) + assert_equal record.merge({"mytag" => "tag.test"}), @d.inject_values_to_record('tag.test', time, record) end test 'injects time as floating point value into specified key as default' do @@ -115,7 +115,7 @@ def config_inject_section(hash = {}) @d.start record = {"key1" => "value1", "key2" => 2} - assert_equal record.merge({"timedata" => float_time}), @d.inject_record('tag', time, record) + assert_equal record.merge({"timedata" => float_time}), @d.inject_values_to_record('tag', time, record) end test 'injects time as unix time into specified key' do @@ -128,7 +128,7 @@ def config_inject_section(hash = {}) @d.start record = {"key1" => "value1", "key2" => 2} - assert_equal record.merge({"timedata" => int_time}), @d.inject_record('tag', time, record) + assert_equal record.merge({"timedata" => int_time}), @d.inject_values_to_record('tag', time, record) end test 'injects time as formatted string in localtime if timezone not specified' do @@ -141,7 +141,7 @@ def config_inject_section(hash = {}) @d.start record = {"key1" => "value1", "key2" => 2} - assert_equal record.merge({"timedata" => "2016_06_21 08:10:11 #{local_timezone}"}), @d.inject_record('tag', time, record) + assert_equal record.merge({"timedata" => "2016_06_21 08:10:11 #{local_timezone}"}), @d.inject_values_to_record('tag', time, record) end test 'injects time as formatted string with nanosecond in localtime if timezone not specified' do @@ -154,7 +154,7 @@ def config_inject_section(hash = {}) @d.start record = {"key1" => "value1", "key2" => 2} - assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320101224 #{local_timezone}"}), @d.inject_record('tag', time, record) + assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320101224 #{local_timezone}"}), @d.inject_values_to_record('tag', time, record) end test 'injects time as formatted string with millisecond in localtime if timezone not specified' do @@ -167,7 +167,7 @@ def config_inject_section(hash = {}) @d.start record = {"key1" => "value1", "key2" => 2} - assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320 #{local_timezone}"}), @d.inject_record('tag', time, record) + assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320 #{local_timezone}"}), @d.inject_values_to_record('tag', time, record) end test 'injects time as formatted string in specified timezone' do @@ -179,7 +179,7 @@ def config_inject_section(hash = {}) @d.start record = {"key1" => "value1", "key2" => 2} - assert_equal record.merge({"timedata" => "2016_06_21 00:10:11 -0800"}), @d.inject_record('tag', time, record) + assert_equal record.merge({"timedata" => "2016_06_21 00:10:11 -0800"}), @d.inject_values_to_record('tag', time, record) end test 'injects hostname, tag and time' do @@ -200,7 +200,7 @@ def config_inject_section(hash = {}) record = {"key1" => "value1", "key2" => 2} injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => "2016_06_20 23:10:11.320101224 +0000"} - assert_equal record.merge(injected), @d.inject_record('tag', time, record) + assert_equal record.merge(injected), @d.inject_values_to_record('tag', time, record) end end sub_test_case 'using inject_event_stream' do @@ -227,7 +227,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end data( @@ -243,7 +243,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end data( @@ -259,7 +259,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end data( @@ -275,7 +275,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end data( @@ -291,7 +291,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end data( @@ -307,7 +307,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end data( @@ -323,7 +323,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end data( @@ -339,7 +339,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end data( @@ -355,7 +355,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end data( @@ -379,7 +379,7 @@ def config_inject_section(hash = {}) data.each do |t, r| expected_es.add(t, r.merge(injected)) end - assert_equal expected_es, @d.inject_event_stream('tag', data) + assert_equal expected_es, @d.inject_values_to_event_stream('tag', data) end end end From e3b8966295bec3d622748fce4362043fca82f129 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 28 Jun 2016 15:17:30 +0900 Subject: [PATCH 17/21] fix to use local/utc time appropriately --- test/plugin_helper/test_inject.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/plugin_helper/test_inject.rb b/test/plugin_helper/test_inject.rb index 36e993b90b..a7ebed3cb6 100644 --- a/test/plugin_helper/test_inject.rb +++ b/test/plugin_helper/test_inject.rb @@ -207,6 +207,9 @@ def config_inject_section(hash = {}) local_timezone = Time.now.strftime('%z') time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i time_subsecond = 320_101_224 + time_in_rational = Rational(time_in_unix * 1_000_000_000 + time_subsecond, 1_000_000_000) + time_in_localtime = Time.at(time_in_rational).localtime + time_in_utc = Time.at(time_in_rational).utc time = Fluent::EventTime.new(time_in_unix, time_subsecond) int_time = 1466464211 float_time = 1466464211.320101 @@ -302,7 +305,7 @@ def config_inject_section(hash = {}) @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z")) @d.start - injected = {"timedata" => "2016_06_21 08:10:11 #{local_timezone}"} + injected = {"timedata" => time_in_localtime.strftime("%Y_%m_%d %H:%M:%S %z")} expected_es = Fluent::MultiEventStream.new data.each do |t, r| expected_es.add(t, r.merge(injected)) @@ -318,7 +321,7 @@ def config_inject_section(hash = {}) @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%N %z")) @d.start - injected = {"timedata" => "2016_06_21 08:10:11.320101224 #{local_timezone}"} + injected = {"timedata" => time_in_localtime.strftime("%Y_%m_%d %H:%M:%S.%N %z")} expected_es = Fluent::MultiEventStream.new data.each do |t, r| expected_es.add(t, r.merge(injected)) @@ -334,7 +337,7 @@ def config_inject_section(hash = {}) @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%3N %z")) @d.start - injected = {"timedata" => "2016_06_21 08:10:11.320 #{local_timezone}"} + injected = {"timedata" => time_in_localtime.strftime("%Y_%m_%d %H:%M:%S.%3N %z")} expected_es = Fluent::MultiEventStream.new data.each do |t, r| expected_es.add(t, r.merge(injected)) @@ -374,7 +377,7 @@ def config_inject_section(hash = {}) )) @d.start - injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => "2016_06_20 23:10:11.320101224 +0000"} + injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => time_in_utc.strftime("%Y_%m_%d %H:%M:%S.%N %z")} expected_es = Fluent::MultiEventStream.new data.each do |t, r| expected_es.add(t, r.merge(injected)) From f40099f8638f723cbc68035b1adb4d1c332d1df1 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 28 Jun 2016 15:36:34 +0900 Subject: [PATCH 18/21] make tests free from timezones --- test/plugin_helper/test_inject.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/plugin_helper/test_inject.rb b/test/plugin_helper/test_inject.rb index a7ebed3cb6..6b24942fde 100644 --- a/test/plugin_helper/test_inject.rb +++ b/test/plugin_helper/test_inject.rb @@ -207,12 +207,11 @@ def config_inject_section(hash = {}) local_timezone = Time.now.strftime('%z') time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i time_subsecond = 320_101_224 + time_micro_second = time_in_unix + 0.320101 time_in_rational = Rational(time_in_unix * 1_000_000_000 + time_subsecond, 1_000_000_000) time_in_localtime = Time.at(time_in_rational).localtime time_in_utc = Time.at(time_in_rational).utc time = Fluent::EventTime.new(time_in_unix, time_subsecond) - int_time = 1466464211 - float_time = 1466464211.320101 data( "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), @@ -273,7 +272,7 @@ def config_inject_section(hash = {}) @d.configure(config_inject_section("time_key" => "timedata")) @d.start - injected = {"timedata" => float_time} + injected = {"timedata" => time_micro_second } expected_es = Fluent::MultiEventStream.new data.each do |t, r| expected_es.add(t, r.merge(injected)) @@ -289,7 +288,7 @@ def config_inject_section(hash = {}) @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime")) @d.start - injected = {"timedata" => int_time} + injected = {"timedata" => time_in_localtime.to_i} expected_es = Fluent::MultiEventStream.new data.each do |t, r| expected_es.add(t, r.merge(injected)) @@ -353,7 +352,7 @@ def config_inject_section(hash = {}) @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z", "timezone" => "-0800")) @d.start - injected = {"timedata" => Time.at(int_time).localtime("-08:00").strftime("%Y_%m_%d %H:%M:%S -0800")} + injected = {"timedata" => Time.at(time_in_unix).localtime("-08:00").strftime("%Y_%m_%d %H:%M:%S -0800")} expected_es = Fluent::MultiEventStream.new data.each do |t, r| expected_es.add(t, r.merge(injected)) From d1ec8dcf5dcdaa5fb8ff3b3660b58bf06f01752e Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 28 Jun 2016 20:55:38 +0900 Subject: [PATCH 19/21] fix to use calculated value as reference instead of fixed value, due to x86/amd64 floating points. Using fixed value (calculated in amd64 environment) is unsafe, because some x86 environment (like Windows 32bit) uses different way to calculate floating point values. It may cause to make tests to fail. --- test/plugin_helper/test_inject.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/plugin_helper/test_inject.rb b/test/plugin_helper/test_inject.rb index 6b24942fde..01b4f696e6 100644 --- a/test/plugin_helper/test_inject.rb +++ b/test/plugin_helper/test_inject.rb @@ -207,11 +207,11 @@ def config_inject_section(hash = {}) local_timezone = Time.now.strftime('%z') time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i time_subsecond = 320_101_224 - time_micro_second = time_in_unix + 0.320101 time_in_rational = Rational(time_in_unix * 1_000_000_000 + time_subsecond, 1_000_000_000) time_in_localtime = Time.at(time_in_rational).localtime time_in_utc = Time.at(time_in_rational).utc time = Fluent::EventTime.new(time_in_unix, time_subsecond) + time_float = time.to_r.to_f data( "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}), @@ -272,7 +272,7 @@ def config_inject_section(hash = {}) @d.configure(config_inject_section("time_key" => "timedata")) @d.start - injected = {"timedata" => time_micro_second } + injected = {"timedata" => time_float } expected_es = Fluent::MultiEventStream.new data.each do |t, r| expected_es.add(t, r.merge(injected)) From b7b3ce40b8e15b66d37d9ab9f6bc13c80a738f5f Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 29 Jun 2016 01:54:36 +0900 Subject: [PATCH 20/21] fix to make test more stable --- test/plugin_helper/test_inject.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/plugin_helper/test_inject.rb b/test/plugin_helper/test_inject.rb index 01b4f696e6..c994d7192e 100644 --- a/test/plugin_helper/test_inject.rb +++ b/test/plugin_helper/test_inject.rb @@ -13,6 +13,7 @@ def config_inject_section(hash = {}) end setup do + Fluent::Test.setup @d = Dummy.new end @@ -79,7 +80,7 @@ def config_inject_section(hash = {}) detected_hostname = `hostname`.chomp @d.configure(config_inject_section("hostname_key" => "host")) logs = @d.log.out.logs - assert{ logs.first && logs.first.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } + assert{ logs.any?{|l| l.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } } @d.start time = event_time() @@ -221,7 +222,7 @@ def config_inject_section(hash = {}) detected_hostname = `hostname`.chomp @d.configure(config_inject_section("hostname_key" => "host")) logs = @d.log.out.logs - assert{ logs.first && logs.first.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } + assert{ logs.any?{|l| l.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } } @d.start injected = {"host" => detected_hostname} From 047dc839198dc3edee667d6f9c60c4b895a03aa6 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Wed, 29 Jun 2016 11:33:06 +0900 Subject: [PATCH 21/21] fix to use micro second value explicitly for floating point time --- lib/fluent/plugin_helper/inject.rb | 2 +- test/plugin_helper/test_inject.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin_helper/inject.rb b/lib/fluent/plugin_helper/inject.rb index 05ab9bc2f6..db9f36ec80 100644 --- a/lib/fluent/plugin_helper/inject.rb +++ b/lib/fluent/plugin_helper/inject.rb @@ -97,7 +97,7 @@ def configure(conf) @_inject_time_key = @inject_config.time_key if @_inject_time_key @_inject_time_formatter = case @inject_config.time_type - when :float then ->(time){ time.to_r.to_f } + when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value when :unixtime then ->(time){ time.to_i } else Fluent::TimeFormatter.new(@inject_config.time_format, false, @inject_config.timezone) diff --git a/test/plugin_helper/test_inject.rb b/test/plugin_helper/test_inject.rb index c994d7192e..4951540280 100644 --- a/test/plugin_helper/test_inject.rb +++ b/test/plugin_helper/test_inject.rb @@ -212,7 +212,7 @@ def config_inject_section(hash = {}) time_in_localtime = Time.at(time_in_rational).localtime time_in_utc = Time.at(time_in_rational).utc time = Fluent::EventTime.new(time_in_unix, time_subsecond) - time_float = time.to_r.to_f + time_float = time.to_r.truncate(+6).to_f data( "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),