diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d4490358bae6..ff3e2f081146 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -244,6 +244,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - add_host_metadata is no GA. {pull}13148[13148] - Add `registered_domain` processor for deriving the registered domain from a given FQDN. {pull}13326[13326] - Add support for RFC3339 time zone offsets in JSON output. {pull}13227[13227] +- Add autodetection mode for add_docker_metadata and enable it by default in included configuration files{pull}13374[13374] *Auditbeat* diff --git a/auditbeat/auditbeat.docker.yml b/auditbeat/auditbeat.docker.yml index 3178297b6f73..a012bbb6aad9 100644 --- a/auditbeat/auditbeat.docker.yml +++ b/auditbeat/auditbeat.docker.yml @@ -14,6 +14,7 @@ auditbeat.modules: - /etc processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml index c0b05dff9e72..ce7d69b97bed 100644 --- a/auditbeat/auditbeat.yml +++ b/auditbeat/auditbeat.yml @@ -148,6 +148,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/filebeat/filebeat.docker.yml b/filebeat/filebeat.docker.yml index 99cf52e1cb66..983e8c5c497b 100644 --- a/filebeat/filebeat.docker.yml +++ b/filebeat/filebeat.docker.yml @@ -5,6 +5,7 @@ filebeat.config: processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index e2aeca00f331..e799f3d9c427 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -176,6 +176,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/heartbeat/heartbeat.docker.yml b/heartbeat/heartbeat.docker.yml index 2302283f674f..59636766d7f6 100644 --- a/heartbeat/heartbeat.docker.yml +++ b/heartbeat/heartbeat.docker.yml @@ -24,6 +24,7 @@ heartbeat.monitors: processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml index ac6a7bfa5e28..6a3749550de1 100644 --- a/heartbeat/heartbeat.yml +++ b/heartbeat/heartbeat.yml @@ -117,9 +117,9 @@ output.elasticsearch: #================================ Processors ===================================== processors: - - add_observer_metadata: + - add_observer_metadata: # Optional, but recommended geo settings for the location Heartbeat is running in - #geo: + #geo: # Token describing this location #name: us-east-1a diff --git a/journalbeat/journalbeat.docker.yml b/journalbeat/journalbeat.docker.yml index a1e67c0d9618..5e723c7a0d24 100644 --- a/journalbeat/journalbeat.docker.yml +++ b/journalbeat/journalbeat.docker.yml @@ -4,6 +4,7 @@ journalbeat.inputs: processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/journalbeat/journalbeat.yml b/journalbeat/journalbeat.yml index a7146f1867d5..b96c8dabe179 100644 --- a/journalbeat/journalbeat.yml +++ b/journalbeat/journalbeat.yml @@ -145,6 +145,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/libbeat/_meta/config.docker.yml b/libbeat/_meta/config.docker.yml index 241ec9906721..6ce79dc1c427 100644 --- a/libbeat/_meta/config.docker.yml +++ b/libbeat/_meta/config.docker.yml @@ -1,5 +1,6 @@ processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/libbeat/_meta/config.yml.tmpl b/libbeat/_meta/config.yml.tmpl index 746f7511c01f..c862ed16c30b 100644 --- a/libbeat/_meta/config.yml.tmpl +++ b/libbeat/_meta/config.yml.tmpl @@ -93,11 +93,12 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ {{else}} processors: - - add_observer_metadata: + - add_observer_metadata: # Optional, but recommended geo settings for the location {{ .BeatName | title }} is running in - #geo: + #geo: # Token describing this location #name: us-east-1a diff --git a/libbeat/common/docker/watcher.go b/libbeat/common/docker/watcher.go index fb862957094d..61b69c37e210 100644 --- a/libbeat/common/docker/watcher.go +++ b/libbeat/common/docker/watcher.go @@ -132,6 +132,12 @@ func NewWatcher(host string, tls *TLSConfig, storeShortID bool) (Watcher, error) return nil, err } + // Extra check to confirm that Docker is available + _, err = client.Info(context.Background()) + if err != nil { + return nil, err + } + return NewWatcherWithClient(client, 60*time.Second, storeShortID) } diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index 24012422fc8f..70b2aaa5e50a 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -57,10 +57,11 @@ type addDockerMetadata struct { fields []string sourceProcessor processors.Processor - pidFields []string // Field names that contain PIDs. - cgroups *common.Cache // Cache of PID (int) to cgropus (map[string]string). - hostFS string // Directory where /proc is found - dedot bool // If set to true, replace dots in labels with `_`. + pidFields []string // Field names that contain PIDs. + cgroups *common.Cache // Cache of PID (int) to cgropus (map[string]string). + hostFS string // Directory where /proc is found + dedot bool // If set to true, replace dots in labels with `_`. + dockerAvailable bool // If Docker exists in env, then it is set to true } // New constructs a new add_docker_metadata processor. @@ -74,13 +75,19 @@ func buildDockerMetadataProcessor(cfg *common.Config, watcherConstructor docker. return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName) } + var dockerAvailable bool + watcher, err := watcherConstructor(config.Host, config.TLS, config.MatchShortID) if err != nil { - return nil, err - } - - if err = watcher.Start(); err != nil { - return nil, err + dockerAvailable = false + errorMsg := fmt.Sprintf("%v: docker environment not detected: %v", processorName, err) + logp.Debug("add_docker_metadata", errorMsg) + } else { + dockerAvailable = true + logp.Debug("add_docker_metadata", "%v: docker environment detected", processorName) + if err = watcher.Start(); err != nil { + return nil, errors.Wrap(err, "failed to start watcher") + } } // Use extract_field processor to get container ID from source file path. @@ -106,6 +113,7 @@ func buildDockerMetadataProcessor(cfg *common.Config, watcherConstructor docker. pidFields: config.MatchPIDs, hostFS: config.HostFS, dedot: config.DeDot, + dockerAvailable: dockerAvailable, }, nil } @@ -121,8 +129,12 @@ func lazyCgroupCacheInit(d *addDockerMetadata) { } func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { + if !d.dockerAvailable { + return event, nil + } var cid string var err error + // Extract CID from the filepath contained in the "log.file.path" field. if d.sourceProcessor != nil { lfp, _ := event.Fields.GetValue("log.file.path") diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index a6973dedbb83..7fc1c00338e7 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -52,6 +52,20 @@ func init() { } } +func TestInitializationNoDocker(t *testing.T) { + var testConfig = common.NewConfig() + testConfig.SetString("host", -1, "unix:///var/run42/docker.sock") + + p, err := buildDockerMetadataProcessor(testConfig, docker.NewWatcher) + assert.NoError(t, err, "initializing add_docker_metadata processor") + + input := common.MapStr{} + result, err := p.Run(&beat.Event{Fields: input}) + assert.NoError(t, err, "processing an event") + + assert.Equal(t, common.MapStr{}, result.Fields) +} + func TestInitialization(t *testing.T) { var testConfig = common.NewConfig() diff --git a/metricbeat/metricbeat.docker.yml b/metricbeat/metricbeat.docker.yml index 982018eefc7a..c34f0f55fcfa 100644 --- a/metricbeat/metricbeat.docker.yml +++ b/metricbeat/metricbeat.docker.yml @@ -4,6 +4,7 @@ metricbeat.config.modules: processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 6f39612eba5d..56038f8f0450 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -120,6 +120,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/packetbeat/packetbeat.docker.yml b/packetbeat/packetbeat.docker.yml index b99d6f670b29..a2a6f8c33b9f 100644 --- a/packetbeat/packetbeat.docker.yml +++ b/packetbeat/packetbeat.docker.yml @@ -39,6 +39,7 @@ packetbeat.protocols.tls: processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 3032364b4a69..a7e18cba72f8 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -202,6 +202,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 7161bc78e77a..84cd4613fbe7 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -127,6 +127,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/auditbeat/auditbeat.docker.yml b/x-pack/auditbeat/auditbeat.docker.yml index 3178297b6f73..a012bbb6aad9 100644 --- a/x-pack/auditbeat/auditbeat.docker.yml +++ b/x-pack/auditbeat/auditbeat.docker.yml @@ -14,6 +14,7 @@ auditbeat.modules: - /etc processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/x-pack/auditbeat/auditbeat.yml b/x-pack/auditbeat/auditbeat.yml index ed5fe117180a..f0e61f8033bf 100644 --- a/x-pack/auditbeat/auditbeat.yml +++ b/x-pack/auditbeat/auditbeat.yml @@ -170,6 +170,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/filebeat/filebeat.docker.yml b/x-pack/filebeat/filebeat.docker.yml index 99cf52e1cb66..983e8c5c497b 100644 --- a/x-pack/filebeat/filebeat.docker.yml +++ b/x-pack/filebeat/filebeat.docker.yml @@ -5,6 +5,7 @@ filebeat.config: processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index e2aeca00f331..e799f3d9c427 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -176,6 +176,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/functionbeat/functionbeat.yml b/x-pack/functionbeat/functionbeat.yml index 19d9da1bbdb4..559f9b581e0f 100644 --- a/x-pack/functionbeat/functionbeat.yml +++ b/x-pack/functionbeat/functionbeat.yml @@ -344,6 +344,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/metricbeat/metricbeat.docker.yml b/x-pack/metricbeat/metricbeat.docker.yml index 982018eefc7a..c34f0f55fcfa 100644 --- a/x-pack/metricbeat/metricbeat.docker.yml +++ b/x-pack/metricbeat/metricbeat.docker.yml @@ -4,6 +4,7 @@ metricbeat.config.modules: processors: - add_cloud_metadata: ~ +- add_docker_metadata: ~ output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' diff --git a/x-pack/metricbeat/metricbeat.yml b/x-pack/metricbeat/metricbeat.yml index 6f39612eba5d..56038f8f0450 100644 --- a/x-pack/metricbeat/metricbeat.yml +++ b/x-pack/metricbeat/metricbeat.yml @@ -120,6 +120,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/winlogbeat/winlogbeat.yml b/x-pack/winlogbeat/winlogbeat.yml index b1771b322e2f..1e1efb1d797d 100644 --- a/x-pack/winlogbeat/winlogbeat.yml +++ b/x-pack/winlogbeat/winlogbeat.yml @@ -139,6 +139,7 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ + - add_docker_metadata: ~ #================================ Logging =====================================