Add autodetection mode for add_docker_metadata

CHANGELOG.next.asciidoc

@@ -244,6 +244,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - add_host_metadata is no GA. {pull}13148[13148]
 - Add `registered_domain` processor for deriving the registered domain from a given FQDN. {pull}13326[13326]
 - Add support for RFC3339 time zone offsets in JSON output. {pull}13227[13227]
+- Add autodetection mode for add_docker_metadata and enable it by default in included configuration files{pull}13374[13374]
 
 *Auditbeat*
 

auditbeat/auditbeat.docker.yml

@@ -14,6 +14,7 @@ auditbeat.modules:
     - /etc
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

auditbeat/auditbeat.yml

@@ -148,6 +148,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

filebeat/filebeat.docker.yml

@@ -5,6 +5,7 @@ filebeat.config:
 
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

filebeat/filebeat.yml

@@ -176,6 +176,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

heartbeat/heartbeat.docker.yml

@@ -24,6 +24,7 @@ heartbeat.monitors:
 
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

heartbeat/heartbeat.yml

@@ -117,9 +117,9 @@ output.elasticsearch:
 #================================ Processors =====================================
 
 processors:
-  - add_observer_metadata: 
+  - add_observer_metadata:
   # Optional, but recommended geo settings for the location Heartbeat is running in
-  #geo: 
+  #geo:
     # Token describing this location
     #name: us-east-1a
 

journalbeat/journalbeat.docker.yml

@@ -4,6 +4,7 @@ journalbeat.inputs:
 
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

journalbeat/journalbeat.yml

@@ -145,6 +145,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

libbeat/_meta/config.docker.yml

@@ -1,5 +1,6 @@
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

libbeat/_meta/config.yml.tmpl

@@ -93,11 +93,12 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 {{else}}
 processors:
-  - add_observer_metadata: 
+  - add_observer_metadata:
   # Optional, but recommended geo settings for the location {{ .BeatName | title }} is running in
-  #geo: 
+  #geo:
     # Token describing this location
     #name: us-east-1a
 

libbeat/common/docker/watcher.go

@@ -132,6 +132,12 @@ func NewWatcher(host string, tls *TLSConfig, storeShortID bool) (Watcher, error)
 		return nil, err
 	}
 
+	// Extra check to confirm that Docker is available
+	_, err = client.Info(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
 	return NewWatcherWithClient(client, 60*time.Second, storeShortID)
 }
 

libbeat/processors/add_docker_metadata/add_docker_metadata.go

@@ -57,10 +57,11 @@ type addDockerMetadata struct {
 	fields          []string
 	sourceProcessor processors.Processor
 
-	pidFields []string      // Field names that contain PIDs.
-	cgroups   *common.Cache // Cache of PID (int) to cgropus (map[string]string).
-	hostFS    string        // Directory where /proc is found
-	dedot     bool          // If set to true, replace dots in labels with `_`.
+	pidFields       []string      // Field names that contain PIDs.
+	cgroups         *common.Cache // Cache of PID (int) to cgropus (map[string]string).
+	hostFS          string        // Directory where /proc is found
+	dedot           bool          // If set to true, replace dots in labels with `_`.
+	dockerAvailable bool          // If Docker exists in env, then it is set to true
 }
 
 // New constructs a new add_docker_metadata processor.
@@ -74,13 +75,19 @@ func buildDockerMetadataProcessor(cfg *common.Config, watcherConstructor docker.
 		return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName)
 	}
 
+	var dockerAvailable bool
+
 	watcher, err := watcherConstructor(config.Host, config.TLS, config.MatchShortID)
 	if err != nil {
-		return nil, err
-	}
-
-	if err = watcher.Start(); err != nil {
-		return nil, err
+		dockerAvailable = false
+		errorMsg := fmt.Sprintf("%v: docker environment not detected: %v", processorName, err)
+		logp.Debug("add_docker_metadata", errorMsg)
+	} else {
+		dockerAvailable = true
+		logp.Debug("add_docker_metadata", "%v: docker environment detected", processorName)
+		if err = watcher.Start(); err != nil {
+			return nil, errors.Wrap(err, "failed to start watcher")
+		}
 	}
 
 	// Use extract_field processor to get container ID from source file path.
@@ -106,6 +113,7 @@ func buildDockerMetadataProcessor(cfg *common.Config, watcherConstructor docker.
 		pidFields:       config.MatchPIDs,
 		hostFS:          config.HostFS,
 		dedot:           config.DeDot,
+		dockerAvailable: dockerAvailable,
 	}, nil
 }
 
@@ -121,8 +129,12 @@ func lazyCgroupCacheInit(d *addDockerMetadata) {
 }
 
 func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) {
+	if !d.dockerAvailable {
+		return event, nil
+	}
 	var cid string
 	var err error
+
 	// Extract CID from the filepath contained in the "log.file.path" field.
 	if d.sourceProcessor != nil {
 		lfp, _ := event.Fields.GetValue("log.file.path")

libbeat/processors/add_docker_metadata/add_docker_metadata_test.go

@@ -52,6 +52,20 @@ func init() {
 	}
 }
 
+func TestInitializationNoDocker(t *testing.T) {
+	var testConfig = common.NewConfig()
+	testConfig.SetString("host", -1, "unix:///var/run42/docker.sock")
+
+	p, err := buildDockerMetadataProcessor(testConfig, docker.NewWatcher)
+	assert.NoError(t, err, "initializing add_docker_metadata processor")
+
+	input := common.MapStr{}
+	result, err := p.Run(&beat.Event{Fields: input})
+	assert.NoError(t, err, "processing an event")
+
+	assert.Equal(t, common.MapStr{}, result.Fields)
+}
+
 func TestInitialization(t *testing.T) {
 	var testConfig = common.NewConfig()
 

metricbeat/metricbeat.docker.yml

@@ -4,6 +4,7 @@ metricbeat.config.modules:
 
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

metricbeat/metricbeat.yml

@@ -120,6 +120,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

packetbeat/packetbeat.docker.yml

@@ -39,6 +39,7 @@ packetbeat.protocols.tls:
 
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

packetbeat/packetbeat.yml

@@ -202,6 +202,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

winlogbeat/winlogbeat.yml

@@ -127,6 +127,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

x-pack/auditbeat/auditbeat.docker.yml

@@ -14,6 +14,7 @@ auditbeat.modules:
     - /etc
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

x-pack/auditbeat/auditbeat.yml

@@ -170,6 +170,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

x-pack/filebeat/filebeat.docker.yml

@@ -5,6 +5,7 @@ filebeat.config:
 
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

x-pack/filebeat/filebeat.yml

@@ -176,6 +176,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

x-pack/functionbeat/functionbeat.yml

@@ -344,6 +344,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

x-pack/metricbeat/metricbeat.docker.yml

@@ -4,6 +4,7 @@ metricbeat.config.modules:
 
 processors:
 - add_cloud_metadata: ~
+- add_docker_metadata: ~
 
 output.elasticsearch:
   hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

x-pack/metricbeat/metricbeat.yml

@@ -120,6 +120,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

x-pack/winlogbeat/winlogbeat.yml

@@ -139,6 +139,7 @@ output.elasticsearch:
 processors:
   - add_host_metadata: ~
   - add_cloud_metadata: ~
+  - add_docker_metadata: ~
 
 #================================ Logging =====================================