Merge pull request #1420 from fluent/json-formatter-without-nl

formatter: Add add_newline parameter to remove '\n' from the result

lib/fluent/plugin/formatter_csv.rb

@@ -29,6 +29,7 @@ class CsvFormatter < Formatter
       # "array" looks good for type of :fields, but this implementation removes tailing comma
       # TODO: Is it needed to support tailing comma?
       config_param :fields, :array, value_type: :string
+      config_param :add_newline, :bool, default: true
 
       def configure(conf)
         super
@@ -42,7 +43,9 @@ def format(tag, time, record)
         row = @fields.map do |key|
           record[key]
         end
-        CSV.generate_line(row, @generate_opts)
+        line = CSV.generate_line(row, @generate_opts)
+        line.chomp! unless @add_newline
+        line
       end
     end
   end

lib/fluent/plugin/formatter_hash.rb

@@ -21,8 +21,12 @@ module Plugin
     class HashFormatter < Formatter
       Plugin.register_formatter('hash', self)
 
+      config_param :add_newline, :bool, default: true
+
       def format(tag, time, record)
-        "#{record.to_s}\n"
+        line = record.to_s
+        line << "\n".freeze if @add_newline
+        line
       end
     end
   end

lib/fluent/plugin/formatter_json.rb

@@ -23,6 +23,7 @@ class JSONFormatter < Formatter
       Plugin.register_formatter('json', self)
 
       config_param :json_parser, :string, default: 'oj'
+      config_param :add_newline, :bool, default: true
 
       def configure(conf)
         super
@@ -35,11 +36,20 @@ def configure(conf)
         rescue LoadError
           @dump_proc = Yajl.method(:dump)
         end
+
+        # format json is used on various highload environment, so re-define method to skip if check
+        unless @add_newline
+          define_singleton_method(:format, method(:format_without_nl))
+        end
       end
 
       def format(tag, time, record)
         "#{@dump_proc.call(record)}\n"
       end
+
+      def format_without_nl(tag, time, record)
+        @dump_proc.call(record)
+      end
     end
   end
 end

lib/fluent/plugin/formatter_ltsv.rb

@@ -25,6 +25,7 @@ class LabeledTSVFormatter < Formatter
 
       config_param :delimiter, :string, default: "\t"
       config_param :label_delimiter, :string, default: ":"
+      config_param :add_newline, :bool, default: true
 
       # TODO: escaping for \t in values
       def format(tag, time, record)
@@ -33,7 +34,7 @@ def format(tag, time, record)
           formatted << @delimiter if formatted.length.nonzero?
           formatted << "#{label}#{@label_delimiter}#{value}"
         end
-        formatted << "\n"
+        formatted << "\n".freeze if @add_newline
         formatted
       end
     end

test/plugin/test_formatter_csv.rb

@@ -54,6 +54,15 @@ def test_format
     assert_equal("\"awesome\",\"awesome2\"\n", formatted)
   end
 
+  def test_format_without_newline
+    d = create_driver("fields" => "message,message2", "add_newline" => false)
+    formatted = d.instance.format(tag, @time, {
+                                    'message' => 'awesome',
+                                    'message2' => 'awesome2'
+                                  })
+    assert_equal("\"awesome\",\"awesome2\"", formatted)
+  end
+
   def test_format_with_customized_delimiters
     d = create_driver("fields" => "message,message2",
                       "delimiter" => "\t")

test/plugin/test_formatter_hash.rb

@@ -0,0 +1,35 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_hash'
+
+class HashFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::HashFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome', 'greeting' => 'hello'}
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}\n!, formatted.encode(Encoding::UTF_8))
+  end
+
+  def test_format_without_newline
+    d = create_driver('add_newline' => false)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}!, formatted.encode(Encoding::UTF_8))
+  end
+end

test/plugin/test_formatter_json.rb

@@ -33,6 +33,14 @@ def test_format(data)
     assert_equal("#{JSON.generate(record)}\n", formatted)
   end
 
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_format_without_nl(data)
+    d = create_driver('json_parser' => data, 'add_newline' => false)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(JSON.generate(record), formatted)
+  end
+
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_format_with_symbolic_record(data)
     d = create_driver('json_parser' => data)

test/plugin/test_formatter_ltsv.rb

@@ -40,6 +40,13 @@ def test_format
     assert_equal("message:awesome\tgreeting:hello\n", formatted)
   end
 
+  def test_format_without_newline
+    d = create_driver('add_newline' => false)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("message:awesome\tgreeting:hello", formatted)
+  end
+
   def test_format_with_customized_delimiters
     d = create_driver(
       'delimiter'       => ',',