From f198ad33fdc1c60823bedde9182156388c0a6c7a Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Wed, 6 Aug 2025 14:25:28 +0530 Subject: [PATCH 01/27] [beatreceivers] Integrate beatsauthextension --- internal/pkg/otel/manager/manager.go | 6 +- internal/pkg/otel/manager/manager_test.go | 3 +- internal/pkg/otel/translate/otelconfig.go | 120 ++++++++++++++---- .../pkg/otel/translate/otelconfig_test.go | 28 +++- 4 files changed, 126 insertions(+), 31 deletions(-) diff --git a/internal/pkg/otel/manager/manager.go b/internal/pkg/otel/manager/manager.go index 06aeb1a2d89..feb61e74d31 100644 --- a/internal/pkg/otel/manager/manager.go +++ b/internal/pkg/otel/manager/manager.go @@ -265,7 +265,7 @@ func (m *OTelManager) Run(ctx context.Context) error { // and reset the retry count m.recoveryTimer.Stop() m.recoveryRetries.Store(0) - mergedCfg, err := buildMergedConfig(cfgUpdate, m.agentInfo, m.beatMonitoringConfigGetter) + mergedCfg, err := buildMergedConfig(cfgUpdate, m.agentInfo, m.beatMonitoringConfigGetter, m.baseLogger) if err != nil { reportErr(ctx, m.errCh, err) continue @@ -297,7 +297,7 @@ func (m *OTelManager) Errors() <-chan error { } // buildMergedConfig combines collector configuration with component-derived configuration. -func buildMergedConfig(cfgUpdate configUpdate, agentInfo info.Agent, monitoringConfigGetter translate.BeatMonitoringConfigGetter) (*confmap.Conf, error) { +func buildMergedConfig(cfgUpdate configUpdate, agentInfo info.Agent, monitoringConfigGetter translate.BeatMonitoringConfigGetter, logger *logp.Logger) (*confmap.Conf, error) { mergedOtelCfg := confmap.New() // Generate component otel config if there are components @@ -305,7 +305,7 @@ func buildMergedConfig(cfgUpdate configUpdate, agentInfo info.Agent, monitoringC if len(cfgUpdate.components) > 0 { model := &component.Model{Components: cfgUpdate.components} var err error - componentOtelCfg, err = translate.GetOtelConfig(model, agentInfo, monitoringConfigGetter) + componentOtelCfg, err = translate.GetOtelConfig(model, agentInfo, monitoringConfigGetter, logger) if err != nil { return nil, fmt.Errorf("failed to generate otel config: %w", err) } diff --git a/internal/pkg/otel/manager/manager_test.go b/internal/pkg/otel/manager/manager_test.go index 215173bbfb3..f59889233ba 100644 --- a/internal/pkg/otel/manager/manager_test.go +++ b/internal/pkg/otel/manager/manager_test.go @@ -30,6 +30,7 @@ import ( "gopkg.in/yaml.v2" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent/pkg/core/logger" "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" @@ -726,7 +727,7 @@ func TestOTelManager_buildMergedConfig(t *testing.T) { collectorCfg: tt.collectorCfg, components: tt.components, } - result, err := buildMergedConfig(cfgUpdate, commonAgentInfo, commonBeatMonitoringConfigGetter) + result, err := buildMergedConfig(cfgUpdate, commonAgentInfo, commonBeatMonitoringConfigGetter, logptest.NewTestingLogger(t, "")) if tt.expectedErrorString != "" { assert.Error(t, err) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index d9ceec48bc0..c3a74816b61 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/transport/httpcommon" "github.com/elastic/elastic-agent/internal/pkg/agent/application/monitoring" @@ -39,7 +40,7 @@ const OtelNamePrefix = "_agent-component/" // BeatMonitoringConfigGetter is a function that returns the monitoring configuration for a beat receiver. type BeatMonitoringConfigGetter func(unitID, binary string) map[string]any -type exporterConfigTranslationFunc func(*config.C) (map[string]any, error) +type exporterConfigTranslationFunc func(*config.C, *logp.Logger) (map[string]any, error) var ( OtelSupportedOutputTypes = []string{"elasticsearch"} @@ -56,6 +57,7 @@ func GetOtelConfig( model *component.Model, info info.Agent, beatMonitoringConfigGetter BeatMonitoringConfigGetter, + logger *logp.Logger, ) (*confmap.Conf, error) { components := getSupportedComponents(model) if len(components) == 0 { @@ -64,7 +66,7 @@ func GetOtelConfig( otelConfig := confmap.New() // base config, nothing here for now for _, comp := range components { - componentConfig, compErr := getCollectorConfigForComponent(comp, info, beatMonitoringConfigGetter) + componentConfig, compErr := getCollectorConfigForComponent(comp, info, beatMonitoringConfigGetter, logger) if compErr != nil { return nil, compErr } @@ -119,15 +121,23 @@ func getExporterID(exporterType otelcomponent.Type, outputName string) otelcompo return otelcomponent.NewIDWithName(exporterType, exporterName) } +// getExtensionID returns the id for beatsauth extension +// outputName here is name of the output defined in elastic-agent.yml. For ex: default, monitoring +func getExtensionID(outputName string) otelcomponent.ID { + extensionName := fmt.Sprintf("%s%s", OtelNamePrefix, outputName) + return otelcomponent.NewIDWithName(otelcomponent.MustNewType("beatsauth"), extensionName) +} + // getCollectorConfigForComponent returns the Otel collector config required to run the given component. // This function returns a full, valid configuration that can then be merged with configurations for other components. func getCollectorConfigForComponent( comp *component.Component, info info.Agent, beatMonitoringConfigGetter BeatMonitoringConfigGetter, + logger *logp.Logger, ) (*confmap.Conf, error) { - exportersConfig, outputQueueConfig, err := getExportersConfigForComponent(comp) + exportersConfig, outputQueueConfig, extensionConfig, err := getExportersConfigForComponent(comp, logger) if err != nil { return nil, err } @@ -154,6 +164,10 @@ func getCollectorConfigForComponent( "pipelines": pipelinesConfig, }, } + + if extensionConfig != nil { + fullConfig["extensions"] = extensionConfig + } return confmap.NewFromStringMap(fullConfig), nil } @@ -256,26 +270,33 @@ func getReceiversConfigForComponent( // getReceiversConfigForComponent returns the exporters configuration and queue settings for a component. Usually this will be a single // exporter, but in principle it could be more. -func getExportersConfigForComponent(comp *component.Component) (exporterCfg map[string]any, queueCfg map[string]any, err error) { +func getExportersConfigForComponent(comp *component.Component, logger *logp.Logger) (exporterCfg map[string]any, queueCfg map[string]any, extensionCfg map[string]any, err error) { exportersConfig := map[string]any{} + extensionConfig := map[string]any{} exporterType, err := getExporterTypeForComponent(comp) if err != nil { - return nil, nil, err + return nil, nil, nil, err } var queueSettings map[string]any for _, unit := range comp.Units { if unit.Type == client.UnitTypeOutput { var unitExportersConfig map[string]any - unitExportersConfig, queueSettings, err = unitToExporterConfig(unit, exporterType, comp.InputType) + var unitExtensionConfig map[string]any + unitExportersConfig, queueSettings, unitExtensionConfig, err = unitToExporterConfig(unit, exporterType, comp.InputType, logger) if err != nil { - return nil, nil, err + return nil, nil, nil, err } for k, v := range unitExportersConfig { exportersConfig[k] = v } + if unitExtensionConfig != nil { + for k, v := range unitExtensionConfig { + extensionConfig[k] = v + } + } } } - return exportersConfig, queueSettings, nil + return exportersConfig, queueSettings, extensionConfig, nil } // GetBeatNameForComponent returns the beat binary name that would be used to run this component. @@ -323,13 +344,13 @@ func getExporterTypeForComponent(comp *component.Component) (otelcomponent.Type, } // unitToExporterConfig translates a component.Unit to return an otel exporter configuration and output queue settings -func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, inputType string) (exportersCfg map[string]any, queueSettings map[string]any, err error) { +func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, inputType string, logger *logp.Logger) (exportersCfg map[string]any, queueSettings map[string]any, extensionCfg map[string]any, err error) { if unit.Type == client.UnitTypeInput { - return nil, nil, fmt.Errorf("unit type is an input, expected output: %v", unit) + return nil, nil, nil, fmt.Errorf("unit type is an input, expected output: %v", unit) } configTranslationFunc, ok := configTranslationFuncForExporter[exporterType] if !ok { - return nil, nil, fmt.Errorf("no config translation function for exporter type: %s", exporterType) + return nil, nil, nil, fmt.Errorf("no config translation function for exporter type: %s", exporterType) } // we'd like to use the same exporter for all outputs with the same name, so we parse out the name for the unit id // these will be deduplicated by the configuration merging process at the end @@ -340,30 +361,53 @@ func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, unitConfigMap := unit.Config.GetSource().AsMap() // this is what beats do in libbeat/management/generate.go outputCfgC, err := config.NewConfigFrom(unitConfigMap) if err != nil { - return nil, nil, fmt.Errorf("error translating config for output: %s, unit: %s, error: %w", outputName, unit.ID, err) + return nil, nil, nil, fmt.Errorf("error translating config for output: %s, unit: %s, error: %w", outputName, unit.ID, err) } + // Config translation function can mutate queue settings defined under output config - exporterConfig, err := configTranslationFunc(outputCfgC) + exporterConfig, err := configTranslationFunc(outputCfgC, logger) if err != nil { - return nil, nil, fmt.Errorf("error translating config for output: %s, unit: %s, error: %w", outputName, unit.ID, err) - } - - exportersCfg = map[string]any{ - exporterId.String(): exporterConfig, + return nil, nil, nil, fmt.Errorf("error translating config for output: %s, unit: %s, error: %w", outputName, unit.ID, err) } // If output config contains queue settings defined by user/preset field, it should be promoted to the receiver section if ok := outputCfgC.HasField("queue"); ok { err := outputCfgC.Unpack(&queueSettings) if err != nil { - return nil, nil, fmt.Errorf("error unpacking queue settings for output: %s, unit: %s, error: %w", outputName, unit.ID, err) + return nil, nil, nil, fmt.Errorf("error unpacking queue settings for output: %s, unit: %s, error: %w", outputName, unit.ID, err) } if queue, ok := queueSettings["queue"].(map[string]any); ok { queueSettings = queue } } - return exportersCfg, queueSettings, nil + // use beatsauth extension for ssl parameters not supported by ES exporter + if exporterType.String() == "elasticsearch" { + // get extension ID + extensionID := getExtensionID(outputName) + extensionConfig, err := getBeatsAuthExtensionConfig(outputCfgC) + if err != nil { + return nil, nil, nil, fmt.Errorf("error supporting ssl parameters for output: %s, unit: %s, error: %w", outputName, unit.ID, err) + } + + if extensionConfig != nil { + // sets extensionCfg + extensionCfg = map[string]any{ + extensionID.String(): extensionConfig, + } + // add authenticator to ES config + exporterConfig["auth"] = map[string]any{ + "authenticator": extensionID.String(), + } + } + + } + + exportersCfg = map[string]any{ + exporterId.String(): exporterConfig, + } + + return exportersCfg, queueSettings, extensionCfg, nil } // getInputsForUnit returns the beat inputs for a unit. These can directly be plugged into a beats receiver config. @@ -410,9 +454,8 @@ func getDefaultDatastreamTypeForComponent(comp *component.Component) (string, er } // translateEsOutputToExporter translates an elasticsearch output configuration to an elasticsearch exporter configuration. -func translateEsOutputToExporter(cfg *config.C) (map[string]any, error) { - // TODO: Figure out a way to avoid needing a logger for this function - esConfig, err := elasticsearchtranslate.ToOTelConfig(cfg, logp.L()) +func translateEsOutputToExporter(cfg *config.C, logger *logp.Logger) (map[string]any, error) { + esConfig, err := elasticsearchtranslate.ToOTelConfig(cfg, logger) if err != nil { return nil, err } @@ -429,3 +472,34 @@ func translateEsOutputToExporter(cfg *config.C) (map[string]any, error) { func BeatDataPath(componentId string) string { return filepath.Join(paths.Run(), componentId) } + +// getBeatsAuthExtensionConfig sets following ssl parameters on beatsauth +// verification_mode +// ca_trusted_fingerprint +// ca_sha256 +func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { + var sslConfig httpcommon.HTTPTransportSettings + err := cfg.Unpack(&sslConfig) + if err != nil { + return nil, err + } + + // if ssl is not enabled or if verification mode is none, then we do not need beatsauth extension + if !sslConfig.TLS.IsEnabled() || sslConfig.TLS.VerificationMode.String() == "none" { + return nil, nil + } + + finalConfig := map[string]any{ + "tls": map[string]any{ + "verification_mode": sslConfig.TLS.VerificationMode.String(), + }, + } + + if len(sslConfig.TLS.CASha256) != 0 { + finalConfig["tls"].(map[string]any)["ca_sha256"] = sslConfig.TLS.CASha256 + } else if sslConfig.TLS.CATrustedFingerprint != "" { + finalConfig["tls"].(map[string]any)["ca_trusted_fingerprint"] = sslConfig.TLS.CATrustedFingerprint + } + + return finalConfig, nil +} diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index ce144b6bc8c..54d4e73f450 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -15,6 +15,7 @@ import ( "go.opentelemetry.io/collector/confmap" "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent/internal/pkg/agent/application/info" @@ -216,6 +217,15 @@ func TestGetOtelConfig(t *testing.T) { "password": "password", "preset": "balanced", "queue.mem.events": 3200, + "ssl.enabled": true, + } + + expectedExtensionConfig := map[string]any{ + "beatsauth/_agent-component/default": map[string]any{ + "tls": map[string]any{ + "verification_mode": "full", + }, + }, } expectedESConfig := map[string]any{ @@ -246,6 +256,13 @@ func TestGetOtelConfig(t *testing.T) { }, "timeout": 90 * time.Second, "idle_conn_timeout": 3 * time.Second, + "auth": map[string]any{ + "authenticator": "beatsauth/_agent-component/default", + }, + "tls": map[string]any{ + "insecure_skip_verify": false, + "include_system_ca_certs_pool": true, + }, }, } @@ -366,7 +383,8 @@ func TestGetOtelConfig(t *testing.T) { }, }, expectedConfig: confmap.NewFromStringMap(map[string]any{ - "exporters": expectedESConfig, + "exporters": expectedESConfig, + "extensions": expectedExtensionConfig, "receivers": map[string]any{ "filebeatreceiver/_agent-component/filestream-default": map[string]any{ "filebeat": map[string]any{ @@ -473,7 +491,8 @@ func TestGetOtelConfig(t *testing.T) { }, }, expectedConfig: confmap.NewFromStringMap(map[string]any{ - "exporters": expectedESConfig, + "exporters": expectedESConfig, + "extensions": expectedExtensionConfig, "receivers": map[string]any{ "metricbeatreceiver/_agent-component/beat-metrics-monitoring": map[string]any{ "metricbeat": map[string]any{ @@ -566,7 +585,8 @@ func TestGetOtelConfig(t *testing.T) { }, }, expectedConfig: confmap.NewFromStringMap(map[string]any{ - "exporters": expectedESConfig, + "exporters": expectedESConfig, + "extensions": expectedExtensionConfig, "receivers": map[string]any{ "metricbeatreceiver/_agent-component/system-metrics": map[string]any{ "metricbeat": map[string]any{ @@ -641,7 +661,7 @@ func TestGetOtelConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actualConf, actualError := GetOtelConfig(tt.model, agentInfo, getBeatMonitoringConfig) + actualConf, actualError := GetOtelConfig(tt.model, agentInfo, getBeatMonitoringConfig, logp.NewNopLogger()) if actualConf == nil || tt.expectedConfig == nil { assert.Equal(t, tt.expectedConfig, actualConf) } else { // this gives a nicer diff From 50047bc91d7b0b232cd2fcd6ac1ba8a19dc7629d Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Wed, 6 Aug 2025 16:18:37 +0530 Subject: [PATCH 02/27] add test cases --- .../pkg/otel/translate/otelconfig_test.go | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index 54d4e73f450..97dda846554 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" + otelcomponent "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" "github.com/elastic/elastic-agent-client/v7/pkg/client" @@ -887,4 +888,136 @@ func TestGetReceiversConfigForComponent(t *testing.T) { } } +func TestBeatsAuthExtension(t *testing.T) { + esInputConfig := map[string]any{ + "type": "elasticsearch", + "hosts": []any{"localhost:9200"}, + "username": "elastic", + "password": "password", + "preset": "balanced", + "queue.mem.events": 3200, + } + + extensionConfig := map[string]any{ + "beatsauth/_agent-component/default": map[string]any{}, + } + + esOutputConfig := map[string]any{ + "elasticsearch/_agent-component/default": map[string]any{ + "batcher": map[string]any{ + "enabled": true, + "max_size": 1600, + "min_size": 0, + }, + "compression": "gzip", + "compression_params": map[string]any{ + "level": 1, + }, + "mapping": map[string]any{ + "mode": "bodymap", + }, + "endpoints": []string{"http://localhost:9200"}, + "password": "password", + "user": "elastic", + "retry": map[string]any{ + "enabled": true, + "initial_interval": 1 * time.Second, + "max_interval": 1 * time.Minute, + "max_retries": 3, + }, + "logs_dynamic_id": map[string]any{ + "enabled": true, + }, + "timeout": 90 * time.Second, + "idle_conn_timeout": 3 * time.Second, + "auth": map[string]any{ + "authenticator": "beatsauth/_agent-component/default", + }, + }, + } + + testCases := []struct { + name string + inputSSLConfig map[string]any + expectedESConfig map[string]any + shouldAuthExist bool + expectedBeatsAuthConfig map[string]any + }{ + { + name: "when ssl.enabled is true", + inputSSLConfig: map[string]any{ + "enabled": true, + }, + expectedESConfig: map[string]any{ + "insecure_skip_verify": false, + "include_system_ca_certs_pool": true, + }, + expectedBeatsAuthConfig: map[string]any{ + "verification_mode": "full", + }, + shouldAuthExist: true, + }, + { + name: "when verification_mode is none", + inputSSLConfig: map[string]any{ + "verification_mode": "none", + }, + expectedESConfig: map[string]any{ + "insecure_skip_verify": true, + "include_system_ca_certs_pool": true, + }, + shouldAuthExist: false, + }, + { + name: "when ca_trusted_fingerprint is set", + inputSSLConfig: map[string]any{ + "verification_mode": "full", + "ca_trusted_fingerprint": "a3:5f:bf:93:12:8f:bc:5c:ab:14:6d:bf:e4:2a:7f:98:9d:2f:16:92:76:c4:12:ab:67:89:fc:56:4b:8e:0c:43", + }, + expectedBeatsAuthConfig: map[string]any{ + "verification_mode": "full", + "ca_trusted_fingerprint": "a3:5f:bf:93:12:8f:bc:5c:ab:14:6d:bf:e4:2a:7f:98:9d:2f:16:92:76:c4:12:ab:67:89:fc:56:4b:8e:0c:43", + }, + shouldAuthExist: true, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + tempMap := esInputConfig + tempMap["ssl"] = test.inputSSLConfig + + units := []component.Unit{ + { + ID: "beat/metrics-default", + Type: client.UnitTypeOutput, + Config: component.MustExpectedConfig(tempMap), + }, + } + + gotES, _, gotBeatsAuth, err := unitToExporterConfig(units[0], otelcomponent.MustNewType("elasticsearch"), "beat/metrics", logp.NewNopLogger()) + if err != nil { + t.Fatal(err) + } + + // if auth should not exist, then ES config should not have auth key + if !test.shouldAuthExist { + expectedES := esOutputConfig + delete(expectedES["elasticsearch/_agent-component/default"].(map[string]any), "auth") + require.Equal(t, expectedES, gotES) + return + } + + expectedES := esOutputConfig + expectedES["elasticsearch/_agent-component/default"].(map[string]any)["tls"] = test.expectedESConfig + require.Equal(t, expectedES, gotES) + + expectedBeatsAuth := extensionConfig + expectedBeatsAuth["beatsauth/_agent-component/default"].(map[string]any)["tls"] = test.expectedBeatsAuthConfig + require.Equal(t, expectedBeatsAuth, gotBeatsAuth) + + }) + } +} + // TODO: Add unit tests for other config generation functions From deeeada738de84f1ae70c9bdbf45a44adb14afc6 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Thu, 14 Aug 2025 18:32:24 +0530 Subject: [PATCH 03/27] final tests and this works --- go.mod | 3 +- go.sum | 4 ++ internal/pkg/otel/components.go | 2 + internal/pkg/otel/translate/otelconfig.go | 15 ++++ .../pkg/otel/translate/otelconfig_test.go | 69 +++++++++++-------- testing/integration/ess/otel_test.go | 7 +- 6 files changed, 70 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index f1a3eb8df6e..4f1a1f78960 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/docker/go-units v0.5.0 github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5 - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812063232-2e0b894269e8 + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812122616-208317d1c011 github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae github.com/elastic/elastic-agent-autodiscover v0.10.0 github.com/elastic/elastic-agent-client/v7 v7.17.2 @@ -32,6 +32,7 @@ require ( github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.16.0 github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.9.0 github.com/elastic/opentelemetry-collector-components/receiver/elasticapmintakereceiver v0.2.0 + github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.1.0 github.com/fatih/color v1.18.0 github.com/fsnotify/fsnotify v1.9.0 github.com/go-viper/mapstructure/v2 v2.3.0 diff --git a/go.sum b/go.sum index 7019191ac8a..64748d04bb1 100644 --- a/go.sum +++ b/go.sum @@ -489,6 +489,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812063232-2e0b894269e8 h1:CCoZ3OxkjwQ9Mg3wm/tMVZ7bJUgMZhsyeX3RL7Cp+J0= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812063232-2e0b894269e8/go.mod h1:7rFXDo5IViULLSZXF5wPCb+lIYXx8/NXH1cT7wtrm5Q= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812122616-208317d1c011 h1:ATkSxddxm00dabggAo0677KODGomM6mOBqa2jL6Za54= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812122616-208317d1c011/go.mod h1:7rFXDo5IViULLSZXF5wPCb+lIYXx8/NXH1cT7wtrm5Q= github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae h1:OiShmbWAyGU0MS0ADJWr1/QgeLIZliMk9xsrFicR3/s= github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae/go.mod h1:D2IckZVXARugvikE4fv1glvaJMohKSZRzrPsxCjo9O0= github.com/elastic/elastic-agent-autodiscover v0.10.0 h1:WJ4zl9uSfk1kHmn2B/0byQBLIL607Zt4LNfOgV7+XN0= @@ -547,6 +549,8 @@ github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextens github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension v0.3.0/go.mod h1:PP7E4hAPU0u2rvfuwe1w2NDyJ5/JH7iziSPUdwxV5+4= github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.5.0 h1:ZMa57/H3ZXC1vkxoXAWP7zPW83NLi672LvxXI0p1fWM= github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.5.0/go.mod h1:7nkWdi3b6ncwuEFZzYmzqX56+V2Qzb5x6jX+DfnQSWI= +github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.1.0 h1:XliuDlQRsZK7wFK0V+x3n2PeFhSOy/8UDW+4AqWg6Rs= +github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.1.0/go.mod h1:i7O6yLRhnMxTVIDGIWRrq1nAz3rgptRHDaiybsG5UF4= github.com/elastic/opentelemetry-collector-components/internal/sharedcomponent v0.0.0-20250220025958-386ba0c4bced h1:XcWi/S3OoeE5Qwmj381AoO3nr3AVXl4Z4QO0mqyjlzU= github.com/elastic/opentelemetry-collector-components/internal/sharedcomponent v0.0.0-20250220025958-386ba0c4bced/go.mod h1:8e9NcGfE2xeor8r+WV9a+hKBEkzJEDnqZN7tqb3GUe8= github.com/elastic/opentelemetry-collector-components/internal/testutil v0.0.0-20250613082151-282de5af1c9b h1:NWuTKdMCJlU9ehRH8V0w1Kk1QI5Vn+9OcJWIO9wI+pE= diff --git a/internal/pkg/otel/components.go b/internal/pkg/otel/components.go index ad22ac4f6f1..2c8ec2140d3 100644 --- a/internal/pkg/otel/components.go +++ b/internal/pkg/otel/components.go @@ -77,6 +77,7 @@ import ( forwardconnector "go.opentelemetry.io/collector/connector/forwardconnector" elasticapmconnector "github.com/elastic/opentelemetry-collector-components/connector/elasticapmconnector" + beatsauthextension "github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension" ) func components(extensionFactories ...extension.Factory) func() (otelcol.Factories, error) { @@ -170,6 +171,7 @@ func components(extensionFactories ...extension.Factory) func() (otelcol.Factori k8sobserver.NewFactory(), apikeyauthextension.NewFactory(), apmconfigextension.NewFactory(), + beatsauthextension.NewFactory(), } extensions = append(extensions, extensionFactories...) factories.Extensions, err = otelcol.MakeFactoryMap[extension.Factory](extensions...) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index c3a74816b61..ab7f81bab14 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -166,8 +166,23 @@ func getCollectorConfigForComponent( } if extensionConfig != nil { + // set extension definition fullConfig["extensions"] = extensionConfig + + // get service map + serviceMap := fullConfig["service"].(map[string]any) + + // we need to convert []string to []interface for this to work + // this is required because otel's merging logic expects []interface + serviceInterface := make([]interface{}, len(maps.Keys(extensionConfig))) + for i, v := range maps.Keys(extensionConfig) { + serviceInterface[i] = v + } + + serviceMap["extensions"] = serviceInterface } + + fmt.Println(confmap.NewFromStringMap(fullConfig)) return confmap.NewFromStringMap(fullConfig), nil } diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index 97dda846554..d78c0ff51b1 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -261,8 +261,9 @@ func TestGetOtelConfig(t *testing.T) { "authenticator": "beatsauth/_agent-component/default", }, "tls": map[string]any{ - "insecure_skip_verify": false, "include_system_ca_certs_pool": true, + "min_version": "1.2", + "max_version": "1.3", }, }, } @@ -451,6 +452,7 @@ func TestGetOtelConfig(t *testing.T) { }, }, "service": map[string]any{ + "extensions": []string{"beatsauth/_agent-component/default"}, "pipelines": map[string]any{ "logs/_agent-component/filestream-default": map[string][]string{ "exporters": []string{"elasticsearch/_agent-component/default"}, @@ -545,6 +547,7 @@ func TestGetOtelConfig(t *testing.T) { }, }, "service": map[string]any{ + "extensions": []string{"beatsauth/_agent-component/default"}, "pipelines": map[string]any{ "logs/_agent-component/beat-metrics-monitoring": map[string][]string{ "exporters": []string{"elasticsearch/_agent-component/default"}, @@ -650,6 +653,7 @@ func TestGetOtelConfig(t *testing.T) { }, }, "service": map[string]any{ + "extensions": []string{"beatsauth/_agent-component/default"}, "pipelines": map[string]any{ "logs/_agent-component/system-metrics": map[string][]string{ "exporters": []string{"elasticsearch/_agent-component/default"}, @@ -939,53 +943,62 @@ func TestBeatsAuthExtension(t *testing.T) { testCases := []struct { name string inputSSLConfig map[string]any - expectedESConfig map[string]any - shouldAuthExist bool + expectedES_TLSConfig map[string]any expectedBeatsAuthConfig map[string]any + shouldAuthExist bool }{ { name: "when ssl.enabled is true", inputSSLConfig: map[string]any{ "enabled": true, }, - expectedESConfig: map[string]any{ - "insecure_skip_verify": false, + expectedES_TLSConfig: map[string]any{ "include_system_ca_certs_pool": true, + "min_version": "1.2", + "max_version": "1.3", }, expectedBeatsAuthConfig: map[string]any{ "verification_mode": "full", }, shouldAuthExist: true, }, - { - name: "when verification_mode is none", - inputSSLConfig: map[string]any{ - "verification_mode": "none", - }, - expectedESConfig: map[string]any{ - "insecure_skip_verify": true, - "include_system_ca_certs_pool": true, - }, - shouldAuthExist: false, - }, { name: "when ca_trusted_fingerprint is set", inputSSLConfig: map[string]any{ "verification_mode": "full", "ca_trusted_fingerprint": "a3:5f:bf:93:12:8f:bc:5c:ab:14:6d:bf:e4:2a:7f:98:9d:2f:16:92:76:c4:12:ab:67:89:fc:56:4b:8e:0c:43", }, + expectedES_TLSConfig: map[string]any{ + "include_system_ca_certs_pool": true, + "min_version": "1.2", + "max_version": "1.3", + }, expectedBeatsAuthConfig: map[string]any{ "verification_mode": "full", "ca_trusted_fingerprint": "a3:5f:bf:93:12:8f:bc:5c:ab:14:6d:bf:e4:2a:7f:98:9d:2f:16:92:76:c4:12:ab:67:89:fc:56:4b:8e:0c:43", }, shouldAuthExist: true, }, + { + name: "when verification_mode is none", + inputSSLConfig: map[string]any{ + "verification_mode": "none", + }, + expectedES_TLSConfig: map[string]any{ + "insecure_skip_verify": true, + "include_system_ca_certs_pool": true, + "min_version": "1.2", + "max_version": "1.3", + }, + shouldAuthExist: false, + }, } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { tempMap := esInputConfig tempMap["ssl"] = test.inputSSLConfig + expectedES := esOutputConfig units := []component.Unit{ { @@ -1000,22 +1013,22 @@ func TestBeatsAuthExtension(t *testing.T) { t.Fatal(err) } - // if auth should not exist, then ES config should not have auth key - if !test.shouldAuthExist { - expectedES := esOutputConfig + // add expected TLS config + expectedES["elasticsearch/_agent-component/default"].(map[string]any)["tls"] = test.expectedES_TLSConfig + + if test.shouldAuthExist { + // check ES exporter config + require.Equal(t, expectedES, gotES) + + // check beats auth config + expectedBeatsAuth := extensionConfig + expectedBeatsAuth["beatsauth/_agent-component/default"].(map[string]any)["tls"] = test.expectedBeatsAuthConfig + require.Equal(t, expectedBeatsAuth, gotBeatsAuth) + } else { // if auth should not exist, then ES config should not have auth key delete(expectedES["elasticsearch/_agent-component/default"].(map[string]any), "auth") require.Equal(t, expectedES, gotES) - return } - expectedES := esOutputConfig - expectedES["elasticsearch/_agent-component/default"].(map[string]any)["tls"] = test.expectedESConfig - require.Equal(t, expectedES, gotES) - - expectedBeatsAuth := extensionConfig - expectedBeatsAuth["beatsauth/_agent-component/default"].(map[string]any)["tls"] = test.expectedBeatsAuthConfig - require.Equal(t, expectedBeatsAuth, gotBeatsAuth) - }) } } diff --git a/testing/integration/ess/otel_test.go b/testing/integration/ess/otel_test.go index 75f81eb270d..5a70740dcb2 100644 --- a/testing/integration/ess/otel_test.go +++ b/testing/integration/ess/otel_test.go @@ -1040,7 +1040,10 @@ func TestOtelFilestreamInput(t *testing.T) { require.True(t, len(esApiKey.Encoded) > 1, "api key is invalid %q", esApiKey) decodedApiKey, err := getDecodedApiKey(esApiKey) require.NoError(t, err) - configTemplate := `inputs: + configTemplate := ` +agent.grpc: + port: 6799 +inputs: - type: filestream id: filestream-e2e use_output: default @@ -1059,6 +1062,8 @@ outputs: hosts: [{{.ESEndpoint}}] api_key: "{{.ESApiKey}}" preset: "balanced" + ssl.enabled: true + ssl.verification_mode: full monitoring: type: elasticsearch hosts: [{{.ESEndpoint}}] From ab4b17f4edb713b1b4e6e6f8c14dc6f646e8c8bd Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 18 Aug 2025 10:07:35 +0530 Subject: [PATCH 04/27] update --- NOTICE-fips.txt | 239 +++++++++++++++++- NOTICE.txt | 239 +++++++++++++++++- go.mod | 20 +- go.sum | 40 +-- .../pkg/otel/translate/otelconfig_test.go | 28 +- wrapper/windows/archive-proxy/go.mod | 2 +- wrapper/windows/archive-proxy/go.sum | 4 +- 7 files changed, 495 insertions(+), 77 deletions(-) diff --git a/NOTICE-fips.txt b/NOTICE-fips.txt index 9d5f8801f7a..c8acccf45d7 100644 --- a/NOTICE-fips.txt +++ b/NOTICE-fips.txt @@ -787,11 +787,11 @@ Contents of probable licence file $GOMODCACHE/github.com/dolmen-go/contextio@v0. -------------------------------------------------------------------------------- Dependency : github.com/elastic/beats/v7 -Version: v7.0.0-alpha2.0.20250812063232-2e0b894269e8 +Version: v7.0.0-alpha2.0.20250814125713-2ac60e4cba71 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20250812063232-2e0b894269e8/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20250814125713-2ac60e4cba71/LICENSE.txt: Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Elastic License. Outside of @@ -3152,6 +3152,217 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.1.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.1.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor Version: v0.16.0 @@ -18389,11 +18600,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.40.0 +Version: v0.41.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.40.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.41.0/LICENSE: Copyright 2009 The Go Authors. @@ -18426,11 +18637,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/exp -Version: v0.0.0-20250215185904-eff6e970281f +Version: v0.0.0-20250811191247-51f88131bc50 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20250215185904-eff6e970281f/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20250811191247-51f88131bc50/LICENSE: Copyright 2009 The Go Authors. @@ -18463,11 +18674,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.42.0 +Version: v0.43.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.42.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.43.0/LICENSE: Copyright 2009 The Go Authors. @@ -18537,11 +18748,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.34.0 +Version: v0.35.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.34.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.35.0/LICENSE: Copyright 2009 The Go Authors. @@ -18574,11 +18785,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.33.0 +Version: v0.34.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.33.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.34.0/LICENSE: Copyright 2009 The Go Authors. @@ -18611,11 +18822,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.27.0 +Version: v0.28.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.27.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.28.0/LICENSE: Copyright 2009 The Go Authors. diff --git a/NOTICE.txt b/NOTICE.txt index 3cdcb90b578..4005a15ad05 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -787,11 +787,11 @@ Contents of probable licence file $GOMODCACHE/github.com/dolmen-go/contextio@v0. -------------------------------------------------------------------------------- Dependency : github.com/elastic/beats/v7 -Version: v7.0.0-alpha2.0.20250812063232-2e0b894269e8 +Version: v7.0.0-alpha2.0.20250814125713-2ac60e4cba71 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20250812063232-2e0b894269e8/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20250814125713-2ac60e4cba71/LICENSE.txt: Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Elastic License. Outside of @@ -3152,6 +3152,217 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.1.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.1.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor Version: v0.16.0 @@ -19022,11 +19233,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.40.0 +Version: v0.41.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.40.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.41.0/LICENSE: Copyright 2009 The Go Authors. @@ -19059,11 +19270,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/exp -Version: v0.0.0-20250215185904-eff6e970281f +Version: v0.0.0-20250811191247-51f88131bc50 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20250215185904-eff6e970281f/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20250811191247-51f88131bc50/LICENSE: Copyright 2009 The Go Authors. @@ -19096,11 +19307,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.42.0 +Version: v0.43.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.42.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.43.0/LICENSE: Copyright 2009 The Go Authors. @@ -19170,11 +19381,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.34.0 +Version: v0.35.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.34.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.35.0/LICENSE: Copyright 2009 The Go Authors. @@ -19207,11 +19418,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.33.0 +Version: v0.34.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.33.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.34.0/LICENSE: Copyright 2009 The Go Authors. @@ -19244,11 +19455,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.27.0 +Version: v0.28.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.27.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.28.0/LICENSE: Copyright 2009 The Go Authors. diff --git a/go.mod b/go.mod index 1c42e70ed7b..c389430f3dc 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/docker/go-units v0.5.0 github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5 - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812122616-208317d1c011 + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250814125713-2ac60e4cba71 github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae github.com/elastic/elastic-agent-autodiscover v0.10.0 github.com/elastic/elastic-agent-client/v7 v7.17.2 @@ -29,10 +29,10 @@ require ( github.com/elastic/opentelemetry-collector-components/connector/elasticapmconnector v0.6.0 github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension v0.3.0 github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.5.0 + github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.1.0 github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.16.0 github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.9.0 github.com/elastic/opentelemetry-collector-components/receiver/elasticapmintakereceiver v0.2.0 - github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.1.0 github.com/fatih/color v1.18.0 github.com/fsnotify/fsnotify v1.9.0 github.com/go-viper/mapstructure/v2 v2.3.0 @@ -86,16 +86,16 @@ require ( go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.130.0 go.opentelemetry.io/collector/receiver/nopreceiver v0.130.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.40.0 - golang.org/x/exp v0.0.0-20250215185904-eff6e970281f - golang.org/x/mod v0.26.0 - golang.org/x/net v0.42.0 + golang.org/x/crypto v0.41.0 + golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 + golang.org/x/mod v0.27.0 + golang.org/x/net v0.43.0 golang.org/x/sync v0.16.0 - golang.org/x/sys v0.34.0 - golang.org/x/term v0.33.0 - golang.org/x/text v0.27.0 + golang.org/x/sys v0.35.0 + golang.org/x/term v0.34.0 + golang.org/x/text v0.28.0 golang.org/x/time v0.12.0 - golang.org/x/tools v0.35.0 + golang.org/x/tools v0.36.0 google.golang.org/api v0.238.0 google.golang.org/grpc v1.74.0 google.golang.org/protobuf v1.36.6 diff --git a/go.sum b/go.sum index 673a85224ae..d08b4dbd2d1 100644 --- a/go.sum +++ b/go.sum @@ -487,10 +487,8 @@ github.com/elastic/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumpti github.com/elastic/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption v1.1.0-elastic/go.mod h1:0vCBR1wgGwZeGmloJ+eCWIZF2S47grTXRzj2mftg2Nk= github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812063232-2e0b894269e8 h1:CCoZ3OxkjwQ9Mg3wm/tMVZ7bJUgMZhsyeX3RL7Cp+J0= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812063232-2e0b894269e8/go.mod h1:7rFXDo5IViULLSZXF5wPCb+lIYXx8/NXH1cT7wtrm5Q= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812122616-208317d1c011 h1:ATkSxddxm00dabggAo0677KODGomM6mOBqa2jL6Za54= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250812122616-208317d1c011/go.mod h1:7rFXDo5IViULLSZXF5wPCb+lIYXx8/NXH1cT7wtrm5Q= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250814125713-2ac60e4cba71 h1:IIHyIzoxKr0ESPZHvrIBr9Q2WrltG5RiuSBqu5Xan3Q= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250814125713-2ac60e4cba71/go.mod h1:HlJgPd4LZbgQ9LAhGZDKd6ewK+yGIqQJoMWk1DuIlVc= github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae h1:OiShmbWAyGU0MS0ADJWr1/QgeLIZliMk9xsrFicR3/s= github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae/go.mod h1:D2IckZVXARugvikE4fv1glvaJMohKSZRzrPsxCjo9O0= github.com/elastic/elastic-agent-autodiscover v0.10.0 h1:WJ4zl9uSfk1kHmn2B/0byQBLIL607Zt4LNfOgV7+XN0= @@ -1524,6 +1522,8 @@ github.com/tidwall/wal v1.1.8 h1:2qDSGdAdjaY3PEvHRva+9UFqgk+ef7cOiW1Qn5JH1y0= github.com/tidwall/wal v1.1.8/go.mod h1:r6lR1j27W9EPalgHiB7zLJDYu3mzW5BQP5KrzBpYY/E= github.com/tilinna/clock v1.1.0 h1:6IQQQCo6KoBxVudv6gwtY8o4eDfhHo8ojA5dP0MfhSs= github.com/tilinna/clock v1.1.0/go.mod h1:ZsP7BcY7sEEz7ktc0IVy8Us6boDrK8VradlKRUGfOao= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= @@ -1903,11 +1903,11 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg= -golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= +golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 h1:3yiSh9fhy5/RhCSntf4Sy0Tnx50DmMpQ4MQdKKk4yg4= +golang.org/x/exp v0.0.0-20250811191247-51f88131bc50/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -1918,8 +1918,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1954,8 +1954,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= @@ -2029,8 +2029,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2043,8 +2043,8 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -2057,8 +2057,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2078,8 +2078,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index d78c0ff51b1..1b38b8ce7a6 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -261,9 +261,8 @@ func TestGetOtelConfig(t *testing.T) { "authenticator": "beatsauth/_agent-component/default", }, "tls": map[string]any{ - "include_system_ca_certs_pool": true, - "min_version": "1.2", - "max_version": "1.3", + "min_version": "1.2", + "max_version": "1.3", }, }, } @@ -452,7 +451,7 @@ func TestGetOtelConfig(t *testing.T) { }, }, "service": map[string]any{ - "extensions": []string{"beatsauth/_agent-component/default"}, + "extensions": []interface{}{"beatsauth/_agent-component/default"}, "pipelines": map[string]any{ "logs/_agent-component/filestream-default": map[string][]string{ "exporters": []string{"elasticsearch/_agent-component/default"}, @@ -547,7 +546,7 @@ func TestGetOtelConfig(t *testing.T) { }, }, "service": map[string]any{ - "extensions": []string{"beatsauth/_agent-component/default"}, + "extensions": []interface{}{"beatsauth/_agent-component/default"}, "pipelines": map[string]any{ "logs/_agent-component/beat-metrics-monitoring": map[string][]string{ "exporters": []string{"elasticsearch/_agent-component/default"}, @@ -653,7 +652,7 @@ func TestGetOtelConfig(t *testing.T) { }, }, "service": map[string]any{ - "extensions": []string{"beatsauth/_agent-component/default"}, + "extensions": []interface{}{"beatsauth/_agent-component/default"}, "pipelines": map[string]any{ "logs/_agent-component/system-metrics": map[string][]string{ "exporters": []string{"elasticsearch/_agent-component/default"}, @@ -953,9 +952,8 @@ func TestBeatsAuthExtension(t *testing.T) { "enabled": true, }, expectedES_TLSConfig: map[string]any{ - "include_system_ca_certs_pool": true, - "min_version": "1.2", - "max_version": "1.3", + "min_version": "1.2", + "max_version": "1.3", }, expectedBeatsAuthConfig: map[string]any{ "verification_mode": "full", @@ -969,9 +967,8 @@ func TestBeatsAuthExtension(t *testing.T) { "ca_trusted_fingerprint": "a3:5f:bf:93:12:8f:bc:5c:ab:14:6d:bf:e4:2a:7f:98:9d:2f:16:92:76:c4:12:ab:67:89:fc:56:4b:8e:0c:43", }, expectedES_TLSConfig: map[string]any{ - "include_system_ca_certs_pool": true, - "min_version": "1.2", - "max_version": "1.3", + "min_version": "1.2", + "max_version": "1.3", }, expectedBeatsAuthConfig: map[string]any{ "verification_mode": "full", @@ -985,10 +982,9 @@ func TestBeatsAuthExtension(t *testing.T) { "verification_mode": "none", }, expectedES_TLSConfig: map[string]any{ - "insecure_skip_verify": true, - "include_system_ca_certs_pool": true, - "min_version": "1.2", - "max_version": "1.3", + "insecure_skip_verify": true, + "min_version": "1.2", + "max_version": "1.3", }, shouldAuthExist: false, }, diff --git a/wrapper/windows/archive-proxy/go.mod b/wrapper/windows/archive-proxy/go.mod index fef9f91e881..c89a23e009f 100644 --- a/wrapper/windows/archive-proxy/go.mod +++ b/wrapper/windows/archive-proxy/go.mod @@ -4,6 +4,6 @@ go 1.24.5 require github.com/elastic/elastic-agent v0.0.0 -require golang.org/x/sys v0.34.0 // indirect +require golang.org/x/sys v0.35.0 // indirect replace github.com/elastic/elastic-agent => ../../../ diff --git a/wrapper/windows/archive-proxy/go.sum b/wrapper/windows/archive-proxy/go.sum index a197cced309..5ace7b9fec4 100644 --- a/wrapper/windows/archive-proxy/go.sum +++ b/wrapper/windows/archive-proxy/go.sum @@ -1,2 +1,2 @@ -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= From 3c0a828fef82245d4e2808511f68bd395bb1744f Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 18 Aug 2025 11:03:50 +0530 Subject: [PATCH 05/27] remove agent.port --- testing/integration/ess/otel_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/testing/integration/ess/otel_test.go b/testing/integration/ess/otel_test.go index 5a70740dcb2..70bb4a5f3ab 100644 --- a/testing/integration/ess/otel_test.go +++ b/testing/integration/ess/otel_test.go @@ -1041,8 +1041,6 @@ func TestOtelFilestreamInput(t *testing.T) { decodedApiKey, err := getDecodedApiKey(esApiKey) require.NoError(t, err) configTemplate := ` -agent.grpc: - port: 6799 inputs: - type: filestream id: filestream-e2e From 1b2b936d4eeb1aa0d3a7c178bf87a504f882e1e4 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 18 Aug 2025 13:25:05 +0530 Subject: [PATCH 06/27] mage otel:readme --- internal/pkg/otel/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/pkg/otel/README.md b/internal/pkg/otel/README.md index 33c9ae476fe..74f286ca8f1 100644 --- a/internal/pkg/otel/README.md +++ b/internal/pkg/otel/README.md @@ -90,6 +90,7 @@ This section provides a summary of components included in the Elastic Distributi | [apikeyauthextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/apikeyauthextension/v0.3.0/extension/apikeyauthextension/README.md) | v0.3.0 | | [apmconfigextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/apmconfigextension/v0.5.0/extension/apmconfigextension/README.md) | v0.5.0 | | [bearertokenauthextension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/bearertokenauthextension/v0.130.0/extension/bearertokenauthextension/README.md) | v0.130.0 | +| [beatsauthextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/beatsauthextension/v0.1.0/extension/beatsauthextension/README.md) | v0.1.0 | | [filestorage](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/storage/filestorage/v0.130.0/extension/storage/filestorage/README.md) | v0.130.0 | | [healthcheckextension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/healthcheckextension/v0.130.0/extension/healthcheckextension/README.md) | v0.130.0 | | [healthcheckv2extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/healthcheckv2extension/v0.130.0/extension/healthcheckv2extension/README.md) | v0.130.0 | From 8cdda2fac37d84fa7f2b49eb0f13b898c249802f Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Tue, 19 Aug 2025 16:41:38 +0530 Subject: [PATCH 07/27] address review comments --- internal/pkg/otel/translate/otelconfig.go | 50 +++++++------------ .../pkg/otel/translate/otelconfig_test.go | 37 ++++++-------- testing/integration/ess/otel_test.go | 2 + 3 files changed, 34 insertions(+), 55 deletions(-) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index ab7f81bab14..ad4d95a1163 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -157,32 +157,22 @@ func getCollectorConfigForComponent( }, } + // we need to convert []string to []interface for this to work + extensionKey := make([]interface{}, len(maps.Keys(extensionConfig))) + for i, v := range maps.Keys(extensionConfig) { + extensionKey[i] = v + } + fullConfig := map[string]any{ - "receivers": receiversConfig, - "exporters": exportersConfig, + "receivers": receiversConfig, + "exporters": exportersConfig, + "extensions": extensionConfig, "service": map[string]any{ - "pipelines": pipelinesConfig, + "extensions": extensionKey, + "pipelines": pipelinesConfig, }, } - if extensionConfig != nil { - // set extension definition - fullConfig["extensions"] = extensionConfig - - // get service map - serviceMap := fullConfig["service"].(map[string]any) - - // we need to convert []string to []interface for this to work - // this is required because otel's merging logic expects []interface - serviceInterface := make([]interface{}, len(maps.Keys(extensionConfig))) - for i, v := range maps.Keys(extensionConfig) { - serviceInterface[i] = v - } - - serviceMap["extensions"] = serviceInterface - } - - fmt.Println(confmap.NewFromStringMap(fullConfig)) return confmap.NewFromStringMap(fullConfig), nil } @@ -304,10 +294,8 @@ func getExportersConfigForComponent(comp *component.Component, logger *logp.Logg for k, v := range unitExportersConfig { exportersConfig[k] = v } - if unitExtensionConfig != nil { - for k, v := range unitExtensionConfig { - extensionConfig[k] = v - } + for k, v := range unitExtensionConfig { + extensionConfig[k] = v } } } @@ -396,7 +384,8 @@ func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, } } - // use beatsauth extension for ssl parameters not supported by ES exporter + // beatsauth extension is not tested with other output types yet + // This extension is used to support ssl parameters if exporterType.String() == "elasticsearch" { // get extension ID extensionID := getExtensionID(outputName) @@ -499,11 +488,6 @@ func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { return nil, err } - // if ssl is not enabled or if verification mode is none, then we do not need beatsauth extension - if !sslConfig.TLS.IsEnabled() || sslConfig.TLS.VerificationMode.String() == "none" { - return nil, nil - } - finalConfig := map[string]any{ "tls": map[string]any{ "verification_mode": sslConfig.TLS.VerificationMode.String(), @@ -512,7 +496,9 @@ func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { if len(sslConfig.TLS.CASha256) != 0 { finalConfig["tls"].(map[string]any)["ca_sha256"] = sslConfig.TLS.CASha256 - } else if sslConfig.TLS.CATrustedFingerprint != "" { + } + + if sslConfig.TLS.CATrustedFingerprint != "" { finalConfig["tls"].(map[string]any)["ca_trusted_fingerprint"] = sslConfig.TLS.CATrustedFingerprint } diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index 1b38b8ce7a6..b0e15926630 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -893,12 +893,11 @@ func TestGetReceiversConfigForComponent(t *testing.T) { func TestBeatsAuthExtension(t *testing.T) { esInputConfig := map[string]any{ - "type": "elasticsearch", - "hosts": []any{"localhost:9200"}, - "username": "elastic", - "password": "password", - "preset": "balanced", - "queue.mem.events": 3200, + "type": "elasticsearch", + "hosts": []any{"localhost:9200"}, + "username": "elastic", + "password": "password", + "preset": "balanced", } extensionConfig := map[string]any{ @@ -944,7 +943,6 @@ func TestBeatsAuthExtension(t *testing.T) { inputSSLConfig map[string]any expectedES_TLSConfig map[string]any expectedBeatsAuthConfig map[string]any - shouldAuthExist bool }{ { name: "when ssl.enabled is true", @@ -958,7 +956,6 @@ func TestBeatsAuthExtension(t *testing.T) { expectedBeatsAuthConfig: map[string]any{ "verification_mode": "full", }, - shouldAuthExist: true, }, { name: "when ca_trusted_fingerprint is set", @@ -974,7 +971,6 @@ func TestBeatsAuthExtension(t *testing.T) { "verification_mode": "full", "ca_trusted_fingerprint": "a3:5f:bf:93:12:8f:bc:5c:ab:14:6d:bf:e4:2a:7f:98:9d:2f:16:92:76:c4:12:ab:67:89:fc:56:4b:8e:0c:43", }, - shouldAuthExist: true, }, { name: "when verification_mode is none", @@ -986,7 +982,9 @@ func TestBeatsAuthExtension(t *testing.T) { "min_version": "1.2", "max_version": "1.3", }, - shouldAuthExist: false, + expectedBeatsAuthConfig: map[string]any{ + "verification_mode": "none", + }, }, } @@ -994,7 +992,6 @@ func TestBeatsAuthExtension(t *testing.T) { t.Run(test.name, func(t *testing.T) { tempMap := esInputConfig tempMap["ssl"] = test.inputSSLConfig - expectedES := esOutputConfig units := []component.Unit{ { @@ -1010,20 +1007,14 @@ func TestBeatsAuthExtension(t *testing.T) { } // add expected TLS config + expectedES := esOutputConfig expectedES["elasticsearch/_agent-component/default"].(map[string]any)["tls"] = test.expectedES_TLSConfig + require.Equal(t, expectedES, gotES) - if test.shouldAuthExist { - // check ES exporter config - require.Equal(t, expectedES, gotES) - - // check beats auth config - expectedBeatsAuth := extensionConfig - expectedBeatsAuth["beatsauth/_agent-component/default"].(map[string]any)["tls"] = test.expectedBeatsAuthConfig - require.Equal(t, expectedBeatsAuth, gotBeatsAuth) - } else { // if auth should not exist, then ES config should not have auth key - delete(expectedES["elasticsearch/_agent-component/default"].(map[string]any), "auth") - require.Equal(t, expectedES, gotES) - } + // check beats auth config + expectedBeatsAuth := extensionConfig + expectedBeatsAuth["beatsauth/_agent-component/default"].(map[string]any)["tls"] = test.expectedBeatsAuthConfig + require.Equal(t, expectedBeatsAuth, gotBeatsAuth) }) } diff --git a/testing/integration/ess/otel_test.go b/testing/integration/ess/otel_test.go index 70bb4a5f3ab..6644983a2e7 100644 --- a/testing/integration/ess/otel_test.go +++ b/testing/integration/ess/otel_test.go @@ -1041,6 +1041,8 @@ func TestOtelFilestreamInput(t *testing.T) { decodedApiKey, err := getDecodedApiKey(esApiKey) require.NoError(t, err) configTemplate := ` +agent.grpc: + port: 6799 inputs: - type: filestream id: filestream-e2e From ba69f1cffae147c407dc06127c054bc199b92695 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Tue, 19 Aug 2025 17:58:17 +0530 Subject: [PATCH 08/27] check is ssl.TLS is non-nil --- internal/pkg/otel/translate/otelconfig.go | 31 +++++++++++++++-------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index ad4d95a1163..7711722699c 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -394,15 +394,13 @@ func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, return nil, nil, nil, fmt.Errorf("error supporting ssl parameters for output: %s, unit: %s, error: %w", outputName, unit.ID, err) } - if extensionConfig != nil { - // sets extensionCfg - extensionCfg = map[string]any{ - extensionID.String(): extensionConfig, - } - // add authenticator to ES config - exporterConfig["auth"] = map[string]any{ - "authenticator": extensionID.String(), - } + // sets extensionCfg + extensionCfg = map[string]any{ + extensionID.String(): extensionConfig, + } + // add authenticator to ES config + exporterConfig["auth"] = map[string]any{ + "authenticator": extensionID.String(), } } @@ -488,9 +486,20 @@ func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { return nil, err } - finalConfig := map[string]any{ + var finalConfig map[string]any + + if sslConfig.TLS == nil { + finalConfig = map[string]any{ + "tls": map[string]any{ + "verification_mode": "full", + }, + } + return finalConfig, nil + } + + finalConfig = map[string]any{ "tls": map[string]any{ - "verification_mode": sslConfig.TLS.VerificationMode.String(), + "verification_mode": sslConfig.TLS.VerificationMode, }, } From b78b572fb5de14df54fec1dc6b90036fb0e53fd7 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Tue, 19 Aug 2025 19:54:20 +0530 Subject: [PATCH 09/27] address review comments --- internal/pkg/otel/translate/otelconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index 7711722699c..4d8304b0345 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -499,7 +499,7 @@ func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { finalConfig = map[string]any{ "tls": map[string]any{ - "verification_mode": sslConfig.TLS.VerificationMode, + "verification_mode": sslConfig.TLS.VerificationMode.String(), }, } From cbd06dd631461a1b343e27ad4b79a29ee3db9a07 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Wed, 20 Aug 2025 06:51:10 +0530 Subject: [PATCH 10/27] remove port --- testing/integration/ess/otel_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/testing/integration/ess/otel_test.go b/testing/integration/ess/otel_test.go index 6644983a2e7..70bb4a5f3ab 100644 --- a/testing/integration/ess/otel_test.go +++ b/testing/integration/ess/otel_test.go @@ -1041,8 +1041,6 @@ func TestOtelFilestreamInput(t *testing.T) { decodedApiKey, err := getDecodedApiKey(esApiKey) require.NoError(t, err) configTemplate := ` -agent.grpc: - port: 6799 inputs: - type: filestream id: filestream-e2e From 1e992c373291cfd1eaaa874b7c87322cd5f0565d Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Thu, 21 Aug 2025 11:54:14 +0530 Subject: [PATCH 11/27] make notice for beatsauth --- NOTICE-fips.txt | 211 ++++++++++++++++++++++++++++++++++++++++++++++++ NOTICE.txt | 211 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 422 insertions(+) diff --git a/NOTICE-fips.txt b/NOTICE-fips.txt index 01a2666d7a2..2851e4afa35 100644 --- a/NOTICE-fips.txt +++ b/NOTICE-fips.txt @@ -3152,6 +3152,217 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.1.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.1.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor Version: v0.16.0 diff --git a/NOTICE.txt b/NOTICE.txt index c7c01357b08..1ae1d39177c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -3152,6 +3152,217 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.1.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.1.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor Version: v0.16.0 From ed573c78f5dc33a75c29a05958cffaee67ed21d2 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 1 Sep 2025 16:07:44 +0530 Subject: [PATCH 12/27] update beatsauthextension --- go.mod | 1 + go.sum | 6 +-- internal/pkg/otel/translate/otelconfig.go | 42 ++++++------------- .../pkg/otel/translate/otelconfig_test.go | 19 ++++++++- 4 files changed, 33 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index a8527208478..e0fe5424ef0 100644 --- a/go.mod +++ b/go.mod @@ -767,6 +767,7 @@ replace ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption => github.com/elastic/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption v1.1.0-elastic github.com/apoydence/eachers => github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 //indirect, see https://github.com/elastic/beats/pull/29780 for details. github.com/dop251/goja => github.com/elastic/goja v0.0.0-20190128172624-dd2ac4456e20 + github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension => github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901101407-7ced08da100d github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/elastic/fsnotify v1.6.1-0.20240920222514-49f82bdbc9e3 github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6 diff --git a/go.sum b/go.sum index 41c43fb37c3..6b40a7f26d9 100644 --- a/go.sum +++ b/go.sum @@ -547,8 +547,6 @@ github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextens github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension v0.3.0/go.mod h1:PP7E4hAPU0u2rvfuwe1w2NDyJ5/JH7iziSPUdwxV5+4= github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.5.0 h1:ZMa57/H3ZXC1vkxoXAWP7zPW83NLi672LvxXI0p1fWM= github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.5.0/go.mod h1:7nkWdi3b6ncwuEFZzYmzqX56+V2Qzb5x6jX+DfnQSWI= -github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.1.0 h1:XliuDlQRsZK7wFK0V+x3n2PeFhSOy/8UDW+4AqWg6Rs= -github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.1.0/go.mod h1:i7O6yLRhnMxTVIDGIWRrq1nAz3rgptRHDaiybsG5UF4= github.com/elastic/opentelemetry-collector-components/internal/sharedcomponent v0.0.0-20250220025958-386ba0c4bced h1:XcWi/S3OoeE5Qwmj381AoO3nr3AVXl4Z4QO0mqyjlzU= github.com/elastic/opentelemetry-collector-components/internal/sharedcomponent v0.0.0-20250220025958-386ba0c4bced/go.mod h1:8e9NcGfE2xeor8r+WV9a+hKBEkzJEDnqZN7tqb3GUe8= github.com/elastic/opentelemetry-collector-components/internal/testutil v0.0.0-20250613082151-282de5af1c9b h1:NWuTKdMCJlU9ehRH8V0w1Kk1QI5Vn+9OcJWIO9wI+pE= @@ -974,6 +972,8 @@ github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901101407-7ced08da100d h1:xpWg8RVVuE21p3Rt2RM2OVskTz73+feERZRsWzAMtOE= +github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901101407-7ced08da100d/go.mod h1:7EPLYwse0In2R+NDLElnHJFMKEFFc+eLRhqyBa3uqlA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= @@ -1522,8 +1522,6 @@ github.com/tidwall/wal v1.1.8 h1:2qDSGdAdjaY3PEvHRva+9UFqgk+ef7cOiW1Qn5JH1y0= github.com/tidwall/wal v1.1.8/go.mod h1:r6lR1j27W9EPalgHiB7zLJDYu3mzW5BQP5KrzBpYY/E= github.com/tilinna/clock v1.1.0 h1:6IQQQCo6KoBxVudv6gwtY8o4eDfhHo8ojA5dP0MfhSs= github.com/tilinna/clock v1.1.0/go.mod h1:ZsP7BcY7sEEz7ktc0IVy8Us6boDrK8VradlKRUGfOao= -github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= -github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index 4d8304b0345..07298cc3d24 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/transport/httpcommon" "github.com/elastic/elastic-agent/internal/pkg/agent/application/monitoring" @@ -23,6 +22,7 @@ import ( "golang.org/x/exp/maps" elasticsearchtranslate "github.com/elastic/beats/v7/libbeat/otelbeat/oteltranslate/outputs/elasticsearch" + "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/x-pack/filebeat/fbreceiver" "github.com/elastic/beats/v7/x-pack/libbeat/management" "github.com/elastic/beats/v7/x-pack/metricbeat/mbreceiver" @@ -391,7 +391,7 @@ func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, extensionID := getExtensionID(outputName) extensionConfig, err := getBeatsAuthExtensionConfig(outputCfgC) if err != nil { - return nil, nil, nil, fmt.Errorf("error supporting ssl parameters for output: %s, unit: %s, error: %w", outputName, unit.ID, err) + return nil, nil, nil, fmt.Errorf("error supporting http parameters for output: %s, unit: %s, error: %w", outputName, unit.ID, err) } // sets extensionCfg @@ -475,41 +475,25 @@ func BeatDataPath(componentId string) string { return filepath.Join(paths.Run(), componentId) } -// getBeatsAuthExtensionConfig sets following ssl parameters on beatsauth -// verification_mode -// ca_trusted_fingerprint -// ca_sha256 +// getBeatsAuthExtensionConfig sets following http parameters on beatsauth +// currently this should only be called for elasticsearch output func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { - var sslConfig httpcommon.HTTPTransportSettings + sslConfig := elasticsearch.ESDefaultTransportSettings() err := cfg.Unpack(&sslConfig) if err != nil { return nil, err } - var finalConfig map[string]any - - if sslConfig.TLS == nil { - finalConfig = map[string]any{ - "tls": map[string]any{ - "verification_mode": "full", - }, - } - return finalConfig, nil - } - - finalConfig = map[string]any{ - "tls": map[string]any{ - "verification_mode": sslConfig.TLS.VerificationMode.String(), - }, - } - - if len(sslConfig.TLS.CASha256) != 0 { - finalConfig["tls"].(map[string]any)["ca_sha256"] = sslConfig.TLS.CASha256 + newConfig, err := config.NewConfigFrom(sslConfig) + if err != nil { + return nil, err } - if sslConfig.TLS.CATrustedFingerprint != "" { - finalConfig["tls"].(map[string]any)["ca_trusted_fingerprint"] = sslConfig.TLS.CATrustedFingerprint + var newMap map[string]any + err = newConfig.Unpack(&newMap) + if err != nil { + return nil, err } - return finalConfig, nil + return newMap, nil } diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index b0e15926630..2f9b9ac9705 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -223,9 +223,24 @@ func TestGetOtelConfig(t *testing.T) { expectedExtensionConfig := map[string]any{ "beatsauth/_agent-component/default": map[string]any{ - "tls": map[string]any{ - "verification_mode": "full", + "idle_connection_timeout": "3s", + "proxy_disable": false, + "ssl": map[string]interface{}{ + "ca_sha256": []interface{}{}, + "ca_trusted_fingerprint": "", + "certificate": "", + "certificate_authorities": []interface{}{}, + "cipher_suites": []interface{}{}, + "curve_types": []interface{}{}, + "enabled": true, + "key": "", + "key_passphrase": "", + "key_passphrase_path": "", + "renegotiation": int64(0), + "supported_protocols": []interface{}{}, + "verification_mode": int64(0), }, + "timeout": "1m30s", }, } From 2fd323c2da998a22bda85c0c4ea487092c178e46 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 1 Sep 2025 19:38:13 +0530 Subject: [PATCH 13/27] beatsauth final test --- NOTICE-fips.txt | 6 +- NOTICE.txt | 6 +- go.mod | 2 +- go.sum | 4 +- internal/pkg/otel/translate/otelconfig.go | 2 +- .../pkg/otel/translate/otelconfig_test.go | 134 +----------------- 6 files changed, 11 insertions(+), 143 deletions(-) diff --git a/NOTICE-fips.txt b/NOTICE-fips.txt index 2851e4afa35..cf30f16569b 100644 --- a/NOTICE-fips.txt +++ b/NOTICE-fips.txt @@ -3153,12 +3153,12 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c -------------------------------------------------------------------------------- -Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension -Version: v0.1.0 +Dependency : github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.0.0-20250901134904-90a34ae77613 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension@v0.0.0-20250901134904-90a34ae77613/LICENSE: Apache License Version 2.0, January 2004 diff --git a/NOTICE.txt b/NOTICE.txt index 1ae1d39177c..3ba1d0b5d7c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -3153,12 +3153,12 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c -------------------------------------------------------------------------------- -Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension -Version: v0.1.0 +Dependency : github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.0.0-20250901134904-90a34ae77613 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension@v0.0.0-20250901134904-90a34ae77613/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index e0fe5424ef0..825692920fb 100644 --- a/go.mod +++ b/go.mod @@ -767,7 +767,7 @@ replace ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption => github.com/elastic/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption v1.1.0-elastic github.com/apoydence/eachers => github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 //indirect, see https://github.com/elastic/beats/pull/29780 for details. github.com/dop251/goja => github.com/elastic/goja v0.0.0-20190128172624-dd2ac4456e20 - github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension => github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901101407-7ced08da100d + github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension => github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901134904-90a34ae77613 github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/elastic/fsnotify v1.6.1-0.20240920222514-49f82bdbc9e3 github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6 diff --git a/go.sum b/go.sum index 6b40a7f26d9..d32c114ee9e 100644 --- a/go.sum +++ b/go.sum @@ -972,8 +972,8 @@ github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901101407-7ced08da100d h1:xpWg8RVVuE21p3Rt2RM2OVskTz73+feERZRsWzAMtOE= -github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901101407-7ced08da100d/go.mod h1:7EPLYwse0In2R+NDLElnHJFMKEFFc+eLRhqyBa3uqlA= +github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901134904-90a34ae77613 h1:DSYCUY/cquHMM+XL1c7xlS7LHhdbuAw3twdH2uqyzbg= +github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901134904-90a34ae77613/go.mod h1:ZQmsSZScCAltjvwh4kxjHuokc7V2+WFLYbNMTAY9wrE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index 07298cc3d24..2905a04d9ab 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -476,7 +476,7 @@ func BeatDataPath(componentId string) string { } // getBeatsAuthExtensionConfig sets following http parameters on beatsauth -// currently this should only be called for elasticsearch output +// currently this is only supported for elasticsearch output func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { sslConfig := elasticsearch.ESDefaultTransportSettings() err := cfg.Unpack(&sslConfig) diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index 2f9b9ac9705..917556f6a30 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/require" - otelcomponent "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" "github.com/elastic/elastic-agent-client/v7/pkg/client" @@ -238,7 +237,7 @@ func TestGetOtelConfig(t *testing.T) { "key_passphrase_path": "", "renegotiation": int64(0), "supported_protocols": []interface{}{}, - "verification_mode": int64(0), + "verification_mode": uint64(0), }, "timeout": "1m30s", }, @@ -905,134 +904,3 @@ func TestGetReceiversConfigForComponent(t *testing.T) { }) } } - -func TestBeatsAuthExtension(t *testing.T) { - esInputConfig := map[string]any{ - "type": "elasticsearch", - "hosts": []any{"localhost:9200"}, - "username": "elastic", - "password": "password", - "preset": "balanced", - } - - extensionConfig := map[string]any{ - "beatsauth/_agent-component/default": map[string]any{}, - } - - esOutputConfig := map[string]any{ - "elasticsearch/_agent-component/default": map[string]any{ - "batcher": map[string]any{ - "enabled": true, - "max_size": 1600, - "min_size": 0, - }, - "compression": "gzip", - "compression_params": map[string]any{ - "level": 1, - }, - "mapping": map[string]any{ - "mode": "bodymap", - }, - "endpoints": []string{"http://localhost:9200"}, - "password": "password", - "user": "elastic", - "retry": map[string]any{ - "enabled": true, - "initial_interval": 1 * time.Second, - "max_interval": 1 * time.Minute, - "max_retries": 3, - }, - "logs_dynamic_id": map[string]any{ - "enabled": true, - }, - "timeout": 90 * time.Second, - "idle_conn_timeout": 3 * time.Second, - "auth": map[string]any{ - "authenticator": "beatsauth/_agent-component/default", - }, - }, - } - - testCases := []struct { - name string - inputSSLConfig map[string]any - expectedES_TLSConfig map[string]any - expectedBeatsAuthConfig map[string]any - }{ - { - name: "when ssl.enabled is true", - inputSSLConfig: map[string]any{ - "enabled": true, - }, - expectedES_TLSConfig: map[string]any{ - "min_version": "1.2", - "max_version": "1.3", - }, - expectedBeatsAuthConfig: map[string]any{ - "verification_mode": "full", - }, - }, - { - name: "when ca_trusted_fingerprint is set", - inputSSLConfig: map[string]any{ - "verification_mode": "full", - "ca_trusted_fingerprint": "a3:5f:bf:93:12:8f:bc:5c:ab:14:6d:bf:e4:2a:7f:98:9d:2f:16:92:76:c4:12:ab:67:89:fc:56:4b:8e:0c:43", - }, - expectedES_TLSConfig: map[string]any{ - "min_version": "1.2", - "max_version": "1.3", - }, - expectedBeatsAuthConfig: map[string]any{ - "verification_mode": "full", - "ca_trusted_fingerprint": "a3:5f:bf:93:12:8f:bc:5c:ab:14:6d:bf:e4:2a:7f:98:9d:2f:16:92:76:c4:12:ab:67:89:fc:56:4b:8e:0c:43", - }, - }, - { - name: "when verification_mode is none", - inputSSLConfig: map[string]any{ - "verification_mode": "none", - }, - expectedES_TLSConfig: map[string]any{ - "insecure_skip_verify": true, - "min_version": "1.2", - "max_version": "1.3", - }, - expectedBeatsAuthConfig: map[string]any{ - "verification_mode": "none", - }, - }, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - tempMap := esInputConfig - tempMap["ssl"] = test.inputSSLConfig - - units := []component.Unit{ - { - ID: "beat/metrics-default", - Type: client.UnitTypeOutput, - Config: component.MustExpectedConfig(tempMap), - }, - } - - gotES, _, gotBeatsAuth, err := unitToExporterConfig(units[0], otelcomponent.MustNewType("elasticsearch"), "beat/metrics", logp.NewNopLogger()) - if err != nil { - t.Fatal(err) - } - - // add expected TLS config - expectedES := esOutputConfig - expectedES["elasticsearch/_agent-component/default"].(map[string]any)["tls"] = test.expectedES_TLSConfig - require.Equal(t, expectedES, gotES) - - // check beats auth config - expectedBeatsAuth := extensionConfig - expectedBeatsAuth["beatsauth/_agent-component/default"].(map[string]any)["tls"] = test.expectedBeatsAuthConfig - require.Equal(t, expectedBeatsAuth, gotBeatsAuth) - - }) - } -} - -// TODO: Add unit tests for other config generation functions From b85e787fb6dede69f67578be691b0228fb3028da Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Tue, 2 Sep 2025 09:56:40 +0530 Subject: [PATCH 14/27] final fix --- NOTICE-fips.txt | 4 ++-- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NOTICE-fips.txt b/NOTICE-fips.txt index cf30f16569b..893503c41c0 100644 --- a/NOTICE-fips.txt +++ b/NOTICE-fips.txt @@ -3154,11 +3154,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c -------------------------------------------------------------------------------- Dependency : github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension -Version: v0.0.0-20250901134904-90a34ae77613 +Version: v0.0.0-20250902040834-316a18b416a5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension@v0.0.0-20250901134904-90a34ae77613/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension@v0.0.0-20250902040834-316a18b416a5/LICENSE: Apache License Version 2.0, January 2004 diff --git a/NOTICE.txt b/NOTICE.txt index 3ba1d0b5d7c..8b93f72f922 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -3154,11 +3154,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c -------------------------------------------------------------------------------- Dependency : github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension -Version: v0.0.0-20250901134904-90a34ae77613 +Version: v0.0.0-20250902040834-316a18b416a5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension@v0.0.0-20250901134904-90a34ae77613/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension@v0.0.0-20250902040834-316a18b416a5/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 825692920fb..ad13246b092 100644 --- a/go.mod +++ b/go.mod @@ -767,7 +767,7 @@ replace ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption => github.com/elastic/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption v1.1.0-elastic github.com/apoydence/eachers => github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 //indirect, see https://github.com/elastic/beats/pull/29780 for details. github.com/dop251/goja => github.com/elastic/goja v0.0.0-20190128172624-dd2ac4456e20 - github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension => github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901134904-90a34ae77613 + github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension => github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250902040834-316a18b416a5 github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/elastic/fsnotify v1.6.1-0.20240920222514-49f82bdbc9e3 github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6 diff --git a/go.sum b/go.sum index d32c114ee9e..d658c8adfad 100644 --- a/go.sum +++ b/go.sum @@ -972,8 +972,8 @@ github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901134904-90a34ae77613 h1:DSYCUY/cquHMM+XL1c7xlS7LHhdbuAw3twdH2uqyzbg= -github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250901134904-90a34ae77613/go.mod h1:ZQmsSZScCAltjvwh4kxjHuokc7V2+WFLYbNMTAY9wrE= +github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250902040834-316a18b416a5 h1:aDznzPuScsl7F6Xp88GZYQAnWUYj01ICRdr9pxtkNbI= +github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250902040834-316a18b416a5/go.mod h1:ZQmsSZScCAltjvwh4kxjHuokc7V2+WFLYbNMTAY9wrE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= From cb2beac6981202f3076367c91214f122c6ba2ed5 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Thu, 4 Sep 2025 16:09:54 +0530 Subject: [PATCH 15/27] add beatsauthextension test --- testing/integration/ess/otel_test.go | 159 +++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/testing/integration/ess/otel_test.go b/testing/integration/ess/otel_test.go index 70bb4a5f3ab..a643ba83095 100644 --- a/testing/integration/ess/otel_test.go +++ b/testing/integration/ess/otel_test.go @@ -10,6 +10,7 @@ import ( "bytes" "context" "encoding/base64" + "encoding/pem" "errors" "fmt" "os" @@ -31,6 +32,7 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/testing/estools" + "github.com/elastic/elastic-agent-libs/transport/tlscommontest" "github.com/elastic/elastic-agent/pkg/control/v2/client" aTesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" @@ -1918,3 +1920,160 @@ service: fixtureWg.Wait() require.True(t, err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded), "Retrieved unexpected error: %s", err.Error()) } + +func TestOtelBeatsAuthExtension(t *testing.T) { + info := define.Require(t, define.Requirements{ + Group: integration.Default, + Local: true, + OS: []define.OS{ + // {Type: define.Windows}, we don't support otel on Windows yet + {Type: define.Linux}, + {Type: define.Darwin}, + }, + Stack: &define.Stack{}, + }) + + // Create the otel configuration file + type otelConfigOptions struct { + ESEndpoint string + ESApiKey string + Index string + CAFile string + } + esEndpoint, err := integration.GetESHost() + require.NoError(t, err, "error getting elasticsearch endpoint") + esApiKey, err := createESApiKey(info.ESClient) + require.NoError(t, err, "error creating API key") + require.True(t, len(esApiKey.Encoded) > 1, "api key is invalid %q", esApiKey) + index := "logs-integration-" + info.Namespace + + fixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) + require.NoError(t, err) + + ctx, cancel := testcontext.WithDeadline(t, t.Context(), time.Now().Add(5*time.Minute)) + defer cancel() + err = fixture.Prepare(ctx) + require.NoError(t, err) + + // create ca-cert + caCert, err := tlscommontest.GenCA() + if err != nil { + t.Fatalf("could not generate root CA certificate: %s", err) + } + + caFilePath := filepath.Join(t.TempDir(), "ca.pem") + os.WriteFile(caFilePath, pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: caCert.Leaf.Raw}), 0o777) + + // we pass an incorrect CA to es-exporter + // but we expect beatsauthextension to replace the exporter's + // roundtripper with beats' implementation of it + // hence expect events to be indexed to elasticsearch + // if authextension is not used - this test fails + otelConfigTemplate := ` +extensions: + beatsauth: + ssl: + enabled: true + verification_mode: none +receivers: + metricbeatreceiver: + metricbeat: + modules: + - module: system + enabled: true + period: 1s + processes: + - '.*' + metricsets: + - cpu + output: + otelconsumer: + queue.mem.flush.timeout: 0s +exporters: + elasticsearch/log: + endpoints: + - {{.ESEndpoint}} + api_key: {{.ESApiKey}} + logs_index: {{.Index}} + batcher: + enabled: true + flush_timeout: 1s + min_size: 1 + tls: + ca_file: {{ .CAFile }} + auth: + authenticator: beatsauth + mapping: + mode: bodymap +service: + extensions: [beatsauth] + pipelines: + logs: + receivers: + - metricbeatreceiver + exporters: + - elasticsearch/log +` + var otelConfigBuffer bytes.Buffer + require.NoError(t, + template.Must(template.New("otelConfig").Parse(otelConfigTemplate)).Execute(&otelConfigBuffer, + otelConfigOptions{ + ESEndpoint: esEndpoint, + ESApiKey: esApiKey.Encoded, + Index: index, + CAFile: caFilePath, + })) + + // configure elastic-agent.yml + err = fixture.Configure(ctx, otelConfigBuffer.Bytes()) + + // prepare agent command + cmd, err := fixture.PrepareAgentCommand(ctx, nil) + require.NoError(t, err, "cannot prepare Elastic-Agent command: %w", err) + + output := strings.Builder{} + cmd.Stderr = &output + cmd.Stdout = &output + + // start elastic-agent + err = cmd.Start() + require.NoError(t, err) + + t.Cleanup(func() { + if t.Failed() { + t.Log("Elastic-Agent output:") + t.Log(output.String()) + } + }) + + require.Eventually(t, func() bool { + err = fixture.IsHealthy(ctx) + if err != nil { + t.Logf("waiting for agent healthy: %s", err.Error()) + return false + } + return true + }, 30*time.Second, 1*time.Second) + + // Make sure find the logs + actualHits := &struct{ Hits int }{} + require.Eventually(t, + func() bool { + findCtx, findCancel := context.WithTimeout(t.Context(), 10*time.Second) + defer findCancel() + + docs, err := estools.GetLogsForIndexWithContext(findCtx, info.ESClient, ".ds-"+index+"*", map[string]interface{}{ + "metricset.name": "cpu", + }) + require.NoError(t, err) + + actualHits.Hits = docs.Hits.Total.Value + return actualHits.Hits >= 1 + }, + 2*time.Minute, 1*time.Second, + "Expected at least %d logs, got %v", 1, actualHits) + + cancel() +} From 24394fb80ab281daf5c06435d92974924ff43a43 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Thu, 4 Sep 2025 16:13:28 +0530 Subject: [PATCH 16/27] Address comments --- internal/pkg/otel/translate/otelconfig.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index 2905a04d9ab..380b07ad7fd 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -121,9 +121,9 @@ func getExporterID(exporterType otelcomponent.Type, outputName string) otelcompo return otelcomponent.NewIDWithName(exporterType, exporterName) } -// getExtensionID returns the id for beatsauth extension +// getBeatsAuthExtensionID returns the id for beatsauth extension // outputName here is name of the output defined in elastic-agent.yml. For ex: default, monitoring -func getExtensionID(outputName string) otelcomponent.ID { +func getBeatsAuthExtensionID(outputName string) otelcomponent.ID { extensionName := fmt.Sprintf("%s%s", OtelNamePrefix, outputName) return otelcomponent.NewIDWithName(otelcomponent.MustNewType("beatsauth"), extensionName) } @@ -388,7 +388,7 @@ func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, // This extension is used to support ssl parameters if exporterType.String() == "elasticsearch" { // get extension ID - extensionID := getExtensionID(outputName) + extensionID := getBeatsAuthExtensionID(outputName) extensionConfig, err := getBeatsAuthExtensionConfig(outputCfgC) if err != nil { return nil, nil, nil, fmt.Errorf("error supporting http parameters for output: %s, unit: %s, error: %w", outputName, unit.ID, err) From 5640e40a09f942048ec027da0540e6904285abdd Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Thu, 4 Sep 2025 16:19:23 +0530 Subject: [PATCH 17/27] fix test --- NOTICE-fips.txt | 4 ++-- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NOTICE-fips.txt b/NOTICE-fips.txt index 9231c2b787f..7e350e9fc39 100644 --- a/NOTICE-fips.txt +++ b/NOTICE-fips.txt @@ -1254,11 +1254,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.22.0 +Version: v0.23.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.22.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.23.2/LICENSE: Apache License Version 2.0, January 2004 diff --git a/NOTICE.txt b/NOTICE.txt index 1d701ff6d17..3e56ecac24b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1254,11 +1254,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.22.0 +Version: v0.23.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.22.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.23.2/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index e3c3101945d..1b9a3fa51a3 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae github.com/elastic/elastic-agent-autodiscover v0.10.0 github.com/elastic/elastic-agent-client/v7 v7.17.2 - github.com/elastic/elastic-agent-libs v0.22.0 + github.com/elastic/elastic-agent-libs v0.23.2 github.com/elastic/elastic-agent-system-metrics v0.11.16 github.com/elastic/elastic-transport-go/v8 v8.7.0 github.com/elastic/go-elasticsearch/v8 v8.19.0 diff --git a/go.sum b/go.sum index da3a916f8ae..a4fcbae9036 100644 --- a/go.sum +++ b/go.sum @@ -497,8 +497,8 @@ github.com/elastic/elastic-agent-autodiscover v0.10.0 h1:WJ4zl9uSfk1kHmn2B/0byQB github.com/elastic/elastic-agent-autodiscover v0.10.0/go.mod h1:Nf3zh9FcJ9nTTswTwDTUAqXmvQllOrNliM6xmORSxwE= github.com/elastic/elastic-agent-client/v7 v7.17.2 h1:Cl2TeABqWZgW40t5fchGWT/sRk4MDDLWA0d8iHHOxLA= github.com/elastic/elastic-agent-client/v7 v7.17.2/go.mod h1:5irRFqp6HLqtu1S+OeY0jg8x7K6PLL+DW+PwVk1vJnk= -github.com/elastic/elastic-agent-libs v0.22.0 h1:36OSYIJ340kzNxwH8bkpSgt3fLbyAPGMj1zD6hcoSok= -github.com/elastic/elastic-agent-libs v0.22.0/go.mod h1:xSeIP3NtOIT4N2pPS4EyURmS1Q8mK0lWZ8Wd1Du6q3w= +github.com/elastic/elastic-agent-libs v0.23.2 h1:ePmOwDyxT2NPHR68QibOV50J/0g00YYQiF120Rodx/M= +github.com/elastic/elastic-agent-libs v0.23.2/go.mod h1:xSeIP3NtOIT4N2pPS4EyURmS1Q8mK0lWZ8Wd1Du6q3w= github.com/elastic/elastic-agent-system-metrics v0.11.16 h1:cLjuO8pE5cUwPGWUHmy1VOERmJVDaep8gY+U4YRQ5vs= github.com/elastic/elastic-agent-system-metrics v0.11.16/go.mod h1:qiZC5p1hd8te4XVnhh7FkXdcYhxFnl5i9GJpROtf6zo= github.com/elastic/elastic-transport-go/v8 v8.7.0 h1:OgTneVuXP2uip4BA658Xi6Hfw+PeIOod2rY3GVMGoVE= From 259eebc5c054145270d4f81a2aa0216dbadea873 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Thu, 4 Sep 2025 16:23:29 +0530 Subject: [PATCH 18/27] fix test --- testing/integration/ess/otel_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/integration/ess/otel_test.go b/testing/integration/ess/otel_test.go index a643ba83095..d074aa5da8e 100644 --- a/testing/integration/ess/otel_test.go +++ b/testing/integration/ess/otel_test.go @@ -1968,8 +1968,8 @@ func TestOtelBeatsAuthExtension(t *testing.T) { // we pass an incorrect CA to es-exporter // but we expect beatsauthextension to replace the exporter's - // roundtripper with beats' implementation of it - // hence expect events to be indexed to elasticsearch + // roundtripper with how beats implements it (with given http configuration block) + // hence we expect events to be indexed to elasticsearch // if authextension is not used - this test fails otelConfigTemplate := ` extensions: From 615190be3f6a28403883f6b4983764558173e0e7 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 8 Sep 2025 15:31:16 +0530 Subject: [PATCH 19/27] change comment --- internal/pkg/otel/translate/otelconfig.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index 380b07ad7fd..dacb6f844f6 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -384,8 +384,7 @@ func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, } } - // beatsauth extension is not tested with other output types yet - // This extension is used to support ssl parameters + // beatsauth extension is not tested with output other than elasticsearch if exporterType.String() == "elasticsearch" { // get extension ID extensionID := getBeatsAuthExtensionID(outputName) From e993d1f6191b791944979138b4a9a0a384eba06c Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Tue, 9 Sep 2025 15:29:44 +0530 Subject: [PATCH 20/27] update beatsauthextension --- NOTICE-fips.txt | 6 +++--- NOTICE.txt | 6 +++--- go.mod | 3 +-- go.sum | 4 ++-- internal/pkg/otel/README.md | 2 +- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/NOTICE-fips.txt b/NOTICE-fips.txt index d079da09cb9..f8dcc3dbe7b 100644 --- a/NOTICE-fips.txt +++ b/NOTICE-fips.txt @@ -3153,12 +3153,12 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c -------------------------------------------------------------------------------- -Dependency : github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension -Version: v0.0.0-20250902040834-316a18b416a5 +Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.2.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension@v0.0.0-20250902040834-316a18b416a5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.2.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/NOTICE.txt b/NOTICE.txt index 83f0d3df93f..d6d053858ea 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -3153,12 +3153,12 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c -------------------------------------------------------------------------------- -Dependency : github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension -Version: v0.0.0-20250902040834-316a18b416a5 +Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.2.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension@v0.0.0-20250902040834-316a18b416a5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.2.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index f0eff2ef9f6..2b3ff642495 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/elastic/opentelemetry-collector-components/connector/elasticapmconnector v0.6.0 github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension v0.3.0 github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.6.0 - github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.1.0 + github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.2.0 github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.16.0 github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.9.0 github.com/elastic/opentelemetry-collector-components/receiver/elasticapmintakereceiver v0.2.1 @@ -769,7 +769,6 @@ replace ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption => github.com/elastic/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption v1.1.0-elastic github.com/apoydence/eachers => github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 //indirect, see https://github.com/elastic/beats/pull/29780 for details. github.com/dop251/goja => github.com/elastic/goja v0.0.0-20190128172624-dd2ac4456e20 - github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension => github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250902040834-316a18b416a5 github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/elastic/fsnotify v1.6.1-0.20240920222514-49f82bdbc9e3 github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6 diff --git a/go.sum b/go.sum index 56f94ff32b0..f3370750cd9 100644 --- a/go.sum +++ b/go.sum @@ -549,6 +549,8 @@ github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextens github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension v0.3.0/go.mod h1:PP7E4hAPU0u2rvfuwe1w2NDyJ5/JH7iziSPUdwxV5+4= github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.6.0 h1:UBAq2kilCpYBKkRQovA8AG5N5AAnQY5IA3ZEhv4vMAo= github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.6.0/go.mod h1:E1uPMjGeBL8PbYCqX4WHMHGqZ3Oo+gK1OKe5Mp1146M= +github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.2.0 h1:cQ4Bu5iyJn5jk68OdwpGIifqVwAZUCoYpN3ZVbVRGBA= +github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.2.0/go.mod h1:aG7w7AA2CydjMxGG6zUZggXCPa+jVRhVYo/92wiDx4Q= github.com/elastic/opentelemetry-collector-components/internal/sharedcomponent v0.0.0-20250220025958-386ba0c4bced h1:XcWi/S3OoeE5Qwmj381AoO3nr3AVXl4Z4QO0mqyjlzU= github.com/elastic/opentelemetry-collector-components/internal/sharedcomponent v0.0.0-20250220025958-386ba0c4bced/go.mod h1:8e9NcGfE2xeor8r+WV9a+hKBEkzJEDnqZN7tqb3GUe8= github.com/elastic/opentelemetry-collector-components/internal/testutil v0.0.0-20250613082151-282de5af1c9b h1:NWuTKdMCJlU9ehRH8V0w1Kk1QI5Vn+9OcJWIO9wI+pE= @@ -974,8 +976,6 @@ github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250902040834-316a18b416a5 h1:aDznzPuScsl7F6Xp88GZYQAnWUYj01ICRdr9pxtkNbI= -github.com/khushijain21/opentelemetry-collector-components/extension/beatsauthextension v0.0.0-20250902040834-316a18b416a5/go.mod h1:ZQmsSZScCAltjvwh4kxjHuokc7V2+WFLYbNMTAY9wrE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= diff --git a/internal/pkg/otel/README.md b/internal/pkg/otel/README.md index d8317949df3..b430b95231b 100644 --- a/internal/pkg/otel/README.md +++ b/internal/pkg/otel/README.md @@ -92,7 +92,7 @@ This section provides a summary of components included in the Elastic Distributi | [apikeyauthextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/apikeyauthextension/v0.3.0/extension/apikeyauthextension/README.md) | v0.3.0 | | [apmconfigextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/apmconfigextension/v0.6.0/extension/apmconfigextension/README.md) | v0.6.0 | | [bearertokenauthextension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/bearertokenauthextension/v0.130.0/extension/bearertokenauthextension/README.md) | v0.130.0 | -| [beatsauthextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/beatsauthextension/v0.1.0/extension/beatsauthextension/README.md) | v0.1.0 | +| [beatsauthextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/beatsauthextension/v0.2.0/extension/beatsauthextension/README.md) | v0.2.0 | | [filestorage](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/storage/filestorage/v0.130.0/extension/storage/filestorage/README.md) | v0.130.0 | | [healthcheckextension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/healthcheckextension/v0.130.0/extension/healthcheckextension/README.md) | v0.130.0 | | [healthcheckv2extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/healthcheckv2extension/v0.130.0/extension/healthcheckv2extension/README.md) | v0.130.0 | From 9a77a5c450e4bf4dfc7e219952948ae38b794c05 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Wed, 10 Sep 2025 09:36:53 +0530 Subject: [PATCH 21/27] Update internal/pkg/otel/translate/otelconfig.go Co-authored-by: Craig MacKenzie --- internal/pkg/otel/translate/otelconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index dacb6f844f6..e013733d7ea 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -477,7 +477,7 @@ func BeatDataPath(componentId string) string { // getBeatsAuthExtensionConfig sets following http parameters on beatsauth // currently this is only supported for elasticsearch output func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { - sslConfig := elasticsearch.ESDefaultTransportSettings() + defaultTransportSettings := elasticsearch.ESDefaultTransportSettings() err := cfg.Unpack(&sslConfig) if err != nil { return nil, err From cc78c68224771fdff850303b6c27712f94b2b500 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Thu, 11 Sep 2025 18:41:20 +0530 Subject: [PATCH 22/27] fix ci --- internal/pkg/otel/translate/otelconfig.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index e013733d7ea..911904f86fc 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -474,16 +474,16 @@ func BeatDataPath(componentId string) string { return filepath.Join(paths.Run(), componentId) } -// getBeatsAuthExtensionConfig sets following http parameters on beatsauth +// getBeatsAuthExtensionConfig sets http transport settings on beatsauth // currently this is only supported for elasticsearch output func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { defaultTransportSettings := elasticsearch.ESDefaultTransportSettings() - err := cfg.Unpack(&sslConfig) + err := cfg.Unpack(&defaultTransportSettings) if err != nil { return nil, err } - newConfig, err := config.NewConfigFrom(sslConfig) + newConfig, err := config.NewConfigFrom(defaultTransportSettings) if err != nil { return nil, err } From b7cb3957e1dd16ebf6b7a2dce8e6b768b014f384 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 12 Sep 2025 13:02:03 +0530 Subject: [PATCH 23/27] add a test that a unique extension is created per output --- internal/pkg/otel/translate/otelconfig.go | 11 +- .../pkg/otel/translate/otelconfig_test.go | 270 ++++++++++++------ 2 files changed, 195 insertions(+), 86 deletions(-) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index 911904f86fc..a5d7a7a702d 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -63,13 +63,21 @@ func GetOtelConfig( if len(components) == 0 { return nil, nil } - otelConfig := confmap.New() // base config, nothing here for now + otelConfig := confmap.New() // base config, nothing here for now + extensionList := []interface{}{} // we have to maintain a list because otel does not merge lists, it overrides them. This is a known issue: see https://github.com/open-telemetry/opentelemetry-collector/issues/8754 for _, comp := range components { componentConfig, compErr := getCollectorConfigForComponent(comp, info, beatMonitoringConfigGetter, logger) if compErr != nil { return nil, compErr } + + if componentConfig.IsSet("service::extensions") { + extensionList = append(extensionList, componentConfig.Get("service::extensions").([]interface{})...) + extensions := confmap.NewFromStringMap(map[string]any{"service::extensions": extensionList}) + componentConfig.Merge(extensions) + } + // the assumption here is that each component will define its own receivers, and the shared exporters // will be merged mergeErr := otelConfig.Merge(componentConfig) @@ -130,6 +138,7 @@ func getBeatsAuthExtensionID(outputName string) otelcomponent.ID { // getCollectorConfigForComponent returns the Otel collector config required to run the given component. // This function returns a full, valid configuration that can then be merged with configurations for other components. +// Note: Lists are not merged and should be handled by the caller of the method func getCollectorConfigForComponent( comp *component.Component, info info.Agent, diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index 917556f6a30..31a115af532 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -221,30 +221,28 @@ func TestGetOtelConfig(t *testing.T) { } expectedExtensionConfig := map[string]any{ - "beatsauth/_agent-component/default": map[string]any{ - "idle_connection_timeout": "3s", - "proxy_disable": false, - "ssl": map[string]interface{}{ - "ca_sha256": []interface{}{}, - "ca_trusted_fingerprint": "", - "certificate": "", - "certificate_authorities": []interface{}{}, - "cipher_suites": []interface{}{}, - "curve_types": []interface{}{}, - "enabled": true, - "key": "", - "key_passphrase": "", - "key_passphrase_path": "", - "renegotiation": int64(0), - "supported_protocols": []interface{}{}, - "verification_mode": uint64(0), - }, - "timeout": "1m30s", + "idle_connection_timeout": "3s", + "proxy_disable": false, + "ssl": map[string]interface{}{ + "ca_sha256": []interface{}{}, + "ca_trusted_fingerprint": "", + "certificate": "", + "certificate_authorities": []interface{}{}, + "cipher_suites": []interface{}{}, + "curve_types": []interface{}{}, + "enabled": true, + "key": "", + "key_passphrase": "", + "key_passphrase_path": "", + "renegotiation": int64(0), + "supported_protocols": []interface{}{}, + "verification_mode": uint64(0), }, + "timeout": "1m30s", } - expectedESConfig := map[string]any{ - "elasticsearch/_agent-component/default": map[string]any{ + expectedESConfig := func(outputName string) map[string]any { + return map[string]any{ "batcher": map[string]any{ "enabled": true, "max_size": 1600, @@ -272,13 +270,13 @@ func TestGetOtelConfig(t *testing.T) { "timeout": 90 * time.Second, "idle_conn_timeout": 3 * time.Second, "auth": map[string]any{ - "authenticator": "beatsauth/_agent-component/default", + "authenticator": "beatsauth/_agent-component/" + outputName, }, "tls": map[string]any{ "min_version": "1.2", "max_version": "1.3", }, - }, + } } defaultProcessors := func(streamId, dataset string, namespace string) []any { @@ -338,6 +336,73 @@ func TestGetOtelConfig(t *testing.T) { } } + // expects input id + expectedFilestreamConfig := func(id string) map[string]any { + return map[string]any{ + "filebeat": map[string]any{ + "inputs": []map[string]any{ + { + "id": "test-1", + "type": "filestream", + "data_stream": map[string]any{ + "dataset": "generic-1", + }, + "paths": []any{ + "/var/log/*.log", + }, + "index": "logs-generic-1-default", + "processors": defaultProcessors("test-1", "generic-1", "logs"), + }, + { + "id": "test-2", + "type": "filestream", + "data_stream": map[string]any{ + "dataset": "generic-2", + }, + "paths": []any{ + "/var/log/*.log", + }, + "index": "logs-generic-2-default", + "processors": defaultProcessors("test-2", "generic-2", "logs"), + }, + }, + }, + "output": map[string]any{ + "otelconsumer": map[string]any{}, + }, + "path": map[string]any{ + "data": filepath.Join(paths.Run(), id), + }, + "queue": map[string]any{ + "mem": map[string]any{ + "events": uint64(3200), + "flush": map[string]any{ + "min_events": uint64(1600), + "timeout": "10s", + }, + }, + }, + "logging": map[string]any{ + "with_fields": map[string]any{ + "component": map[string]any{ + "binary": "filebeat", + "dataset": "elastic_agent.filebeat", + "type": "filestream", + "id": id, + }, + "log": map[string]any{ + "source": id, + }, + }, + }, + "http": map[string]any{ + "enabled": true, + "host": "localhost", + }, + } + + } + getBeatMonitoringConfig := func(_, _ string) map[string]any { return map[string]any{ "http": map[string]any{ @@ -398,78 +463,105 @@ func TestGetOtelConfig(t *testing.T) { }, }, expectedConfig: confmap.NewFromStringMap(map[string]any{ - "exporters": expectedESConfig, - "extensions": expectedExtensionConfig, + "exporters": map[string]any{ + "elasticsearch/_agent-component/default": expectedESConfig("default"), + }, + "extensions": map[string]any{ + "beatsauth/_agent-component/default": expectedExtensionConfig, + }, "receivers": map[string]any{ - "filebeatreceiver/_agent-component/filestream-default": map[string]any{ - "filebeat": map[string]any{ - "inputs": []map[string]any{ - { - "id": "test-1", - "type": "filestream", - "data_stream": map[string]any{ - "dataset": "generic-1", - }, - "paths": []any{ - "/var/log/*.log", - }, - "index": "logs-generic-1-default", - "processors": defaultProcessors("test-1", "generic-1", "logs"), - }, - { - "id": "test-2", - "type": "filestream", - "data_stream": map[string]any{ - "dataset": "generic-2", - }, - "paths": []any{ - "/var/log/*.log", - }, - "index": "logs-generic-2-default", - "processors": defaultProcessors("test-2", "generic-2", "logs"), + "filebeatreceiver/_agent-component/filestream-default": expectedFilestreamConfig("filestream-default"), + }, + "service": map[string]any{ + "extensions": []interface{}{"beatsauth/_agent-component/default"}, + "pipelines": map[string]any{ + "logs/_agent-component/filestream-default": map[string][]string{ + "exporters": []string{"elasticsearch/_agent-component/default"}, + "receivers": []string{"filebeatreceiver/_agent-component/filestream-default"}, + }, + }, + }, + }), + }, + { + name: "multiple filestream inputs and output types", + model: &component.Model{ + Components: []component.Component{ + { + ID: "filestream-primaryOutput", + InputType: "filestream", + OutputType: "elasticsearch", + InputSpec: &component.InputRuntimeSpec{ + BinaryName: "agentbeat", + Spec: component.InputSpec{ + Command: &component.CommandSpec{ + Args: []string{"filebeat"}, }, }, }, - "output": map[string]any{ - "otelconsumer": map[string]any{}, - }, - "path": map[string]any{ - "data": filepath.Join(paths.Run(), "filestream-default"), - }, - "queue": map[string]any{ - "mem": map[string]any{ - "events": uint64(3200), - "flush": map[string]any{ - "min_events": uint64(1600), - "timeout": "10s", - }, + Units: []component.Unit{ + { + ID: "filestream-unit", + Type: client.UnitTypeInput, + Config: component.MustExpectedConfig(fileStreamConfig), + }, + { + ID: "filestream-primaryOutput", + Type: client.UnitTypeOutput, + Config: component.MustExpectedConfig(esOutputConfig), }, }, - "logging": map[string]any{ - "with_fields": map[string]any{ - "component": map[string]any{ - "binary": "filebeat", - "dataset": "elastic_agent.filebeat", - "type": "filestream", - "id": "filestream-default", - }, - "log": map[string]any{ - "source": "filestream-default", + }, + { + ID: "filestream-secondaryOutput", + InputType: "filestream", + OutputType: "elasticsearch", + InputSpec: &component.InputRuntimeSpec{ + BinaryName: "agentbeat", + Spec: component.InputSpec{ + Command: &component.CommandSpec{ + Args: []string{"filebeat"}, }, }, }, - "http": map[string]any{ - "enabled": true, - "host": "localhost", + Units: []component.Unit{ + { + ID: "filestream-unit-2", + Type: client.UnitTypeInput, + Config: component.MustExpectedConfig(fileStreamConfig), + }, + { + ID: "filestream-secondaryOutput", + Type: client.UnitTypeOutput, + Config: component.MustExpectedConfig(esOutputConfig), + }, }, }, }, + }, + expectedConfig: confmap.NewFromStringMap(map[string]any{ + "exporters": map[string]any{ + "elasticsearch/_agent-component/primaryOutput": expectedESConfig("primaryOutput"), + "elasticsearch/_agent-component/secondaryOutput": expectedESConfig("secondaryOutput"), + }, + "extensions": map[string]any{ + "beatsauth/_agent-component/primaryOutput": expectedExtensionConfig, + "beatsauth/_agent-component/secondaryOutput": expectedExtensionConfig, + }, + "receivers": map[string]any{ + "filebeatreceiver/_agent-component/filestream-primaryOutput": expectedFilestreamConfig("filestream-primaryOutput"), + "filebeatreceiver/_agent-component/filestream-secondaryOutput": expectedFilestreamConfig("filestream-secondaryOutput"), + }, "service": map[string]any{ - "extensions": []interface{}{"beatsauth/_agent-component/default"}, + "extensions": []interface{}{"beatsauth/_agent-component/primaryOutput", "beatsauth/_agent-component/secondaryOutput"}, "pipelines": map[string]any{ - "logs/_agent-component/filestream-default": map[string][]string{ - "exporters": []string{"elasticsearch/_agent-component/default"}, - "receivers": []string{"filebeatreceiver/_agent-component/filestream-default"}, + "logs/_agent-component/filestream-primaryOutput": map[string][]string{ + "exporters": []string{"elasticsearch/_agent-component/primaryOutput"}, + "receivers": []string{"filebeatreceiver/_agent-component/filestream-primaryOutput"}, + }, + "logs/_agent-component/filestream-secondaryOutput": map[string][]string{ + "exporters": []string{"elasticsearch/_agent-component/secondaryOutput"}, + "receivers": []string{"filebeatreceiver/_agent-component/filestream-secondaryOutput"}, }, }, }, @@ -507,8 +599,12 @@ func TestGetOtelConfig(t *testing.T) { }, }, expectedConfig: confmap.NewFromStringMap(map[string]any{ - "exporters": expectedESConfig, - "extensions": expectedExtensionConfig, + "exporters": map[string]any{ + "elasticsearch/_agent-component/default": expectedESConfig("default"), + }, + "extensions": map[string]any{ + "beatsauth/_agent-component/default": expectedExtensionConfig, + }, "receivers": map[string]any{ "metricbeatreceiver/_agent-component/beat-metrics-monitoring": map[string]any{ "metricbeat": map[string]any{ @@ -602,8 +698,12 @@ func TestGetOtelConfig(t *testing.T) { }, }, expectedConfig: confmap.NewFromStringMap(map[string]any{ - "exporters": expectedESConfig, - "extensions": expectedExtensionConfig, + "exporters": map[string]any{ + "elasticsearch/_agent-component/default": expectedESConfig("default"), + }, + "extensions": map[string]any{ + "beatsauth/_agent-component/default": expectedExtensionConfig, + }, "receivers": map[string]any{ "metricbeatreceiver/_agent-component/system-metrics": map[string]any{ "metricbeat": map[string]any{ From 73d216cc327ee3fa1190177f1c3359c14b1a0677 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 12 Sep 2025 13:06:05 +0530 Subject: [PATCH 24/27] fix ci --- internal/pkg/otel/translate/otelconfig.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index a5d7a7a702d..e4e602b6a51 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -75,7 +75,10 @@ func GetOtelConfig( if componentConfig.IsSet("service::extensions") { extensionList = append(extensionList, componentConfig.Get("service::extensions").([]interface{})...) extensions := confmap.NewFromStringMap(map[string]any{"service::extensions": extensionList}) - componentConfig.Merge(extensions) + err := componentConfig.Merge(extensions) + if err != nil { + return nil, fmt.Errorf("error merging otel extensions for component %s: %w", comp.ID, err) + } } // the assumption here is that each component will define its own receivers, and the shared exporters From caaf19965b0ffc540f70d49e8cf359a2cd599612 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 12 Sep 2025 13:06:34 +0530 Subject: [PATCH 25/27] add comment --- internal/pkg/otel/translate/otelconfig.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index e4e602b6a51..6bef6f4a6ae 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -72,6 +72,7 @@ func GetOtelConfig( return nil, compErr } + // logic to merge extension list if componentConfig.IsSet("service::extensions") { extensionList = append(extensionList, componentConfig.Get("service::extensions").([]interface{})...) extensions := confmap.NewFromStringMap(map[string]any{"service::extensions": extensionList}) From f61a6e8be508724e7bff3de7e5c29c11131c4f6f Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 15 Sep 2025 14:31:58 +0530 Subject: [PATCH 26/27] add better test --- .../pkg/otel/translate/otelconfig_test.go | 101 +++++++++++------- 1 file changed, 64 insertions(+), 37 deletions(-) diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index 31a115af532..486ecb9c177 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -210,35 +210,62 @@ func TestGetOtelConfig(t *testing.T) { }, }, } - esOutputConfig := map[string]any{ - "type": "elasticsearch", - "hosts": []any{"localhost:9200"}, - "username": "elastic", - "password": "password", - "preset": "balanced", - "queue.mem.events": 3200, - "ssl.enabled": true, + + type extraParams struct { + key string + value any } + // pass ssl params as extra args to this method + esOutputConfig := func(extra ...extraParams) map[string]any { + finalOutput := map[string]any{ + "type": "elasticsearch", + "hosts": []any{"localhost:9200"}, + "username": "elastic", + "password": "password", + "preset": "balanced", + "queue.mem.events": 3200, + "ssl.enabled": true, + } - expectedExtensionConfig := map[string]any{ - "idle_connection_timeout": "3s", - "proxy_disable": false, - "ssl": map[string]interface{}{ - "ca_sha256": []interface{}{}, - "ca_trusted_fingerprint": "", - "certificate": "", - "certificate_authorities": []interface{}{}, - "cipher_suites": []interface{}{}, - "curve_types": []interface{}{}, - "enabled": true, - "key": "", - "key_passphrase": "", - "key_passphrase_path": "", - "renegotiation": int64(0), - "supported_protocols": []interface{}{}, - "verification_mode": uint64(0), - }, - "timeout": "1m30s", + for _, v := range extra { + finalOutput[v.key] = v.value + } + return finalOutput + } + + expectedExtensionConfig := func(extra ...extraParams) map[string]any { + finalOutput := map[string]any{ + "idle_connection_timeout": "3s", + "proxy_disable": false, + "ssl": map[string]interface{}{ + "ca_sha256": []interface{}{}, + "ca_trusted_fingerprint": "", + "certificate": "", + "certificate_authorities": []interface{}{}, + "cipher_suites": []interface{}{}, + "curve_types": []interface{}{}, + "enabled": true, + "key": "", + "key_passphrase": "", + "key_passphrase_path": "", + "renegotiation": int64(0), + "supported_protocols": []interface{}{}, + "verification_mode": uint64(0), + }, + "timeout": "1m30s", + } + for _, v := range extra { + // accepts one level deep parameters to replace + if _, ok := v.value.(map[string]any); ok { + for newkey, newvalue := range v.value.(map[string]any) { + // this is brittle - it is expected that developers will pass expected params correctly here + finalOutput[v.key].(map[string]any)[newkey] = newvalue + } + continue + } + finalOutput[v.key] = v.value + } + return finalOutput } expectedESConfig := func(outputName string) map[string]any { @@ -456,7 +483,7 @@ func TestGetOtelConfig(t *testing.T) { { ID: "filestream-default", Type: client.UnitTypeOutput, - Config: component.MustExpectedConfig(esOutputConfig), + Config: component.MustExpectedConfig(esOutputConfig()), }, }, }, @@ -467,7 +494,7 @@ func TestGetOtelConfig(t *testing.T) { "elasticsearch/_agent-component/default": expectedESConfig("default"), }, "extensions": map[string]any{ - "beatsauth/_agent-component/default": expectedExtensionConfig, + "beatsauth/_agent-component/default": expectedExtensionConfig(), }, "receivers": map[string]any{ "filebeatreceiver/_agent-component/filestream-default": expectedFilestreamConfig("filestream-default"), @@ -508,7 +535,7 @@ func TestGetOtelConfig(t *testing.T) { { ID: "filestream-primaryOutput", Type: client.UnitTypeOutput, - Config: component.MustExpectedConfig(esOutputConfig), + Config: component.MustExpectedConfig(esOutputConfig(extraParams{"ssl.verification_mode", "certificate"})), }, }, }, @@ -533,7 +560,7 @@ func TestGetOtelConfig(t *testing.T) { { ID: "filestream-secondaryOutput", Type: client.UnitTypeOutput, - Config: component.MustExpectedConfig(esOutputConfig), + Config: component.MustExpectedConfig(esOutputConfig(extraParams{"ssl.ca_trusted_fingerprint", "b9a10bbe64ee9826abeda6546fc988c8bf798b41957c33d05db736716513dc9c"})), }, }, }, @@ -545,8 +572,8 @@ func TestGetOtelConfig(t *testing.T) { "elasticsearch/_agent-component/secondaryOutput": expectedESConfig("secondaryOutput"), }, "extensions": map[string]any{ - "beatsauth/_agent-component/primaryOutput": expectedExtensionConfig, - "beatsauth/_agent-component/secondaryOutput": expectedExtensionConfig, + "beatsauth/_agent-component/primaryOutput": expectedExtensionConfig(extraParams{"ssl", map[string]any{"verification_mode": uint64(2)}}), + "beatsauth/_agent-component/secondaryOutput": expectedExtensionConfig(extraParams{"ssl", map[string]any{"ca_trusted_fingerprint": "b9a10bbe64ee9826abeda6546fc988c8bf798b41957c33d05db736716513dc9c"}}), }, "receivers": map[string]any{ "filebeatreceiver/_agent-component/filestream-primaryOutput": expectedFilestreamConfig("filestream-primaryOutput"), @@ -592,7 +619,7 @@ func TestGetOtelConfig(t *testing.T) { { ID: "beat/metrics-default", Type: client.UnitTypeOutput, - Config: component.MustExpectedConfig(esOutputConfig), + Config: component.MustExpectedConfig(esOutputConfig()), }, }, }, @@ -603,7 +630,7 @@ func TestGetOtelConfig(t *testing.T) { "elasticsearch/_agent-component/default": expectedESConfig("default"), }, "extensions": map[string]any{ - "beatsauth/_agent-component/default": expectedExtensionConfig, + "beatsauth/_agent-component/default": expectedExtensionConfig(), }, "receivers": map[string]any{ "metricbeatreceiver/_agent-component/beat-metrics-monitoring": map[string]any{ @@ -691,7 +718,7 @@ func TestGetOtelConfig(t *testing.T) { { ID: "system/metrics-default", Type: client.UnitTypeOutput, - Config: component.MustExpectedConfig(esOutputConfig), + Config: component.MustExpectedConfig(esOutputConfig()), }, }, }, @@ -702,7 +729,7 @@ func TestGetOtelConfig(t *testing.T) { "elasticsearch/_agent-component/default": expectedESConfig("default"), }, "extensions": map[string]any{ - "beatsauth/_agent-component/default": expectedExtensionConfig, + "beatsauth/_agent-component/default": expectedExtensionConfig(), }, "receivers": map[string]any{ "metricbeatreceiver/_agent-component/system-metrics": map[string]any{ From 9c0a6b7886a05ad507635642dbb32d14421159b5 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 15 Sep 2025 14:41:39 +0530 Subject: [PATCH 27/27] fix test --- internal/pkg/otel/translate/otelconfig_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index b4719a2cb8b..2e765c33e59 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -270,11 +270,6 @@ func TestGetOtelConfig(t *testing.T) { expectedESConfig := func(outputName string) map[string]any { return map[string]any{ - "batcher": map[string]any{ - "enabled": true, - "max_size": 1600, - "min_size": 0, - }, "compression": "gzip", "compression_params": map[string]any{ "level": 1,