diff --git a/internal/pkg/agent/application/monitoring/testdata/monitoring_config_full.yaml b/internal/pkg/agent/application/monitoring/testdata/monitoring_config_full.yaml index 7ee1db80671..70fb1595afd 100644 --- a/internal/pkg/agent/application/monitoring/testdata/monitoring_config_full.yaml +++ b/internal/pkg/agent/application/monitoring/testdata/monitoring_config_full.yaml @@ -32,6 +32,10 @@ inputs: when: regexp: message: ^Non-zero metrics in the last + - drop_event: + when: + regexp: + input: \{"create":\{"_index":.* - copy_fields: fields: - from: data_stream.dataset diff --git a/internal/pkg/agent/application/monitoring/v1_monitor.go b/internal/pkg/agent/application/monitoring/v1_monitor.go index 911e86bb4b9..2be5926d049 100644 --- a/internal/pkg/agent/application/monitoring/v1_monitor.go +++ b/internal/pkg/agent/application/monitoring/v1_monitor.go @@ -831,6 +831,8 @@ func processorsForAgentFilestream() []any { dropEventsFromMonitoringComponentsProcessor(), // drop periodic metrics logs (those are useful mostly in diagnostic dumps where we collect log files) dropPeriodicMetricsLogsProcessor(), + // drop sensitive information from ES exporter ensuring we do not send it to fleet + dropSensitiveInfoFromESExporter(), } // if the event is from a component, use the component's dataset processors = append(processors, useComponentDatasetProcessors()...) @@ -1152,6 +1154,19 @@ func dropEcsVersionFieldProcessor() map[string]any { } } +// dropSensitiveInfoFromESExporter returns a processor which drops any sensitive information logged by ES exporter +func dropSensitiveInfoFromESExporter() map[string]any { + return map[string]interface{}{ + "drop_event": map[string]interface{}{ + "when": map[string]interface{}{ + "regexp": map[string]interface{}{ + "input": `\{"create":\{"_index":.*`, + }, + }, + }, + } +} + // addFormattedIndexProcessor returns a processor which sets the destination index for an event based on a format string. func addFormattedIndexProcessor() map[string]any { return map[string]any{ diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index bada4abc252..0288ed8ea32 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -401,12 +401,14 @@ func translateEsOutputToExporter(cfg *config.C) (map[string]any, error) { if err != nil { return nil, err } + + // logs failed documents at debug level + esConfig["telemetry"] = map[string]any{ + "log_failed_docs_input": true, + } // dynamic indexing works by default // we also want to use dynamic log ids esConfig["logs_dynamic_id"] = map[string]any{"enabled": true} - - // for compatibility with beats, we want bodymap mapping - esConfig["mapping"] = map[string]any{"mode": "bodymap"} return esConfig, nil }