Keep original messages in case of Filebeat modules

CHANGELOG.asciidoc

@@ -19,6 +19,8 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 
 *Filebeat*
 
+- Keep original messages in case of Filebeat modules. {pull}8448[8448]
+
 *Heartbeat*
 
 *Metricbeat*

filebeat/_meta/fields.common.yml

@@ -112,6 +112,12 @@
       description: >
         This field contains the flags of the event.
 
+    - name: log.message
+      type: keyword
+      description: >
+        The unprocessed original log message. This can be used for reprocessing logs.
+      index: false
+
     - name: event.created
       type: date
       description: >

filebeat/channel/factory.go

@@ -20,6 +20,7 @@ package channel
 import (
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/processors"
 )
 
@@ -43,6 +44,9 @@ type clientEventer struct {
 // inputOutletConfig defines common input settings
 // for the publisher pipeline.
 type inputOutletConfig struct {
+	// KeepOriginalMsg determines if the original message needs to be kept for a module.
+	KeepOriginalMsg bool `config:"keep_original_message"`
+
 	// event processing
 	common.EventMetadata `config:",inline"`      // Fields and tags to add to events.
 	Processors           processors.PluginConfig `config:"processors"`
@@ -59,6 +63,10 @@ type inputOutletConfig struct {
 
 }
 
+var defaultConfig = inputOutletConfig{
+	KeepOriginalMsg: true,
+}
+
 // NewOutletFactory creates a new outlet factory for
 // connecting an input to the publisher pipeline.
 func NewOutletFactory(
@@ -82,7 +90,7 @@ func NewOutletFactory(
 // This guarantees ordering between events as required by the registrar for
 // file.State updates
 func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *common.MapStrPointer) (Outleter, error) {
-	config := inputOutletConfig{}
+	config := defaultConfig
 	if err := cfg.Unpack(&config); err != nil {
 		return nil, err
 	}
@@ -101,13 +109,16 @@ func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *c
 	meta := common.MapStr{}
 	setMeta(meta, "pipeline", config.Pipeline)
 
+	keepOriginal := false
 	fields := common.MapStr{}
 	setMeta(fields, "module", config.Module)
 	setMeta(fields, "name", config.Fileset)
 	if len(fields) > 0 {
 		fields = common.MapStr{
 			"fileset": fields,
 		}
+		keepOriginal = config.KeepOriginalMsg
+
 	}
 	if config.Type != "" {
 		fields["prospector"] = common.MapStr{
@@ -119,13 +130,14 @@ func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *c
 	}
 
 	client, err := p.ConnectWith(beat.ClientConfig{
-		PublishMode:   beat.GuaranteedSend,
-		EventMetadata: config.EventMetadata,
-		DynamicFields: dynFields,
-		Meta:          meta,
-		Fields:        fields,
-		Processor:     processors,
-		Events:        f.eventer,
+		PublishMode:     beat.GuaranteedSend,
+		EventMetadata:   config.EventMetadata,
+		DynamicFields:   dynFields,
+		Meta:            meta,
+		Fields:          fields,
+		KeepOriginalMsg: keepOriginal,
+		Processor:       processors,
+		Events:          f.eventer,
 	})
 	if err != nil {
 		return nil, err

filebeat/docs/fields.asciidoc

@@ -3042,6 +3042,18 @@ Logging level.
 This field contains the flags of the event.
 
 
+--
+
+*`log.message`*::
++
+--
+type: keyword
+
+The unprocessed original log message. This can be used for reprocessing logs.
+
+
+Field is not indexed.
+
 --
 
 *`event.created`*::

filebeat/filebeat.reference.yml

@@ -27,6 +27,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
   # Authorization logs
   #auth:
@@ -42,6 +45,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 #------------------------------- Apache2 Module ------------------------------
 #- module: apache2
@@ -56,6 +62,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
   # Error logs
   #error:
@@ -68,6 +77,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 #------------------------------- Auditd Module -------------------------------
 #- module: auditd
@@ -81,6 +93,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 #---------------------------- elasticsearch Module ---------------------------
 - module: elasticsearch
@@ -142,6 +157,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
   # Debug logs
   #debug:
@@ -154,6 +172,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
   # Startup logs
   #startup:
@@ -166,6 +187,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 #--------------------------------- IIS Module --------------------------------
 #- module: iis
@@ -180,6 +204,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
   # Error logs
   #error:
@@ -192,6 +219,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 #-------------------------------- Kafka Module -------------------------------
 - module: kafka
@@ -250,6 +280,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 #-------------------------------- MySQL Module -------------------------------
 #- module: mysql
@@ -264,6 +297,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
   # Slow logs
   #slowlog:
@@ -276,6 +312,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 #-------------------------------- Nginx Module -------------------------------
 #- module: nginx
@@ -302,6 +341,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 #------------------------------- Osquery Module ------------------------------
 - module: osquery
@@ -330,6 +372,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 #-------------------------------- Redis Module -------------------------------
 #- module: redis
@@ -364,6 +409,9 @@ filebeat.modules:
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+      #Keeps the original message, so the data can be processed again on Ingest Node
+      #It requires increased storage size, because the sizes of events are approximately doubled.
+      #keep_original_message: true
 
 
 #=========================== Filebeat inputs =============================