diff --git a/.chloggen/esexporter-queuebatch.yaml b/.chloggen/esexporter-queuebatch.yaml new file mode 100644 index 0000000000000..4060ceb62137b --- /dev/null +++ b/.chloggen/esexporter-queuebatch.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: deprecation + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Allow `sending_queue` to queue and batch based on the new API and deprecate `batcher` config + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [41338] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/.chloggen/mx-psi_configoptional-exporter-pulsar.yaml b/.chloggen/mx-psi_configoptional-exporter-pulsar.yaml new file mode 100644 index 0000000000000..4fab3e4f83385 --- /dev/null +++ b/.chloggen/mx-psi_configoptional-exporter-pulsar.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pulsarexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Use `configoptional.Optional` for authentication fields + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [41723] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [api] diff --git a/.chloggen/prometheusexporter-translation-strategies.yaml b/.chloggen/prometheusexporter-translation-strategies.yaml new file mode 100644 index 0000000000000..5671cbf192bc2 --- /dev/null +++ b/.chloggen/prometheusexporter-translation-strategies.yaml @@ -0,0 +1,34 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: exporter/prometheus + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add `translation_strategy` configuration option to control how OTLP metric names are translated to Prometheus format. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35459] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + The new `translation_strategy` option provides four different translation modes: + - `UnderscoreEscapingWithSuffixes`: Escapes special characters to underscores and appends type/unit suffixes + - `UnderscoreEscapingWithoutSuffixes`: Escapes special characters but omits suffixes + - `NoUTF8EscapingWithSuffixes`: Preserves UTF-8 characters while adding suffixes + - `NoTranslation`: Passes metric names through unaltered + When `translation_strategy` is set, it always takes precedence over the deprecated `add_metric_suffixes` option. + The `exporter.prometheusexporter.DisableAddMetricSuffixes` feature gate can be used to completely ignore the deprecated `add_metric_suffixes` setting. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/connector/datadogconnector/go.mod b/connector/datadogconnector/go.mod index 486c9e8a31616..031cafb65a6d4 100644 --- a/connector/datadogconnector/go.mod +++ b/connector/datadogconnector/go.mod @@ -122,7 +122,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/statstracker v0.66.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.67.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 // indirect github.com/DataDog/datadog-agent/pkg/version v0.68.0 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.43.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect diff --git a/connector/datadogconnector/go.sum b/connector/datadogconnector/go.sum index 02196eea04508..6388bed56b987 100644 --- a/connector/datadogconnector/go.sum +++ b/connector/datadogconnector/go.sum @@ -209,8 +209,8 @@ github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 h1:pM5p51BfBYgkp github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0/go.mod h1:cQWaDB4GxK3i+0jbXZlePXzJ3f1oz2bnEPVfF9rK1Dg= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0 h1:mLXT+D29nRHZdUHQvLp+mskZ9kQZJqkkk05EsEjw7WY= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0/go.mod h1:yi2rztz2PWPdqIykyyX4nHkJiQLvEn0dTPzj9V7g7o8= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 h1:as8nOTMEeifcLCeHN+oqfdRrtUtVZIvwIN7kD2fQDIM= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0/go.mod h1:2l1nTFRygeIQkYr8E1tHuO/ZMjxPt0VgJ1wW14F2Fb4= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 h1:8euFCQn/SNpuN3L5uHx0bHj3N6Yf4QBukUSdylwArSA= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608/go.mod h1:mKV8FnTNGRSJF/TEPDOYWCCnarpPsxhfTZBVkFh/zRY= github.com/DataDog/datadog-agent/pkg/version v0.68.0 h1:O9K/SxuDrKJp+juA724SS1STQxw7meylHWTHjlHs+rM= github.com/DataDog/datadog-agent/pkg/version v0.68.0/go.mod h1:zpRXbtsHTgxP3vMyL6hsKQt/c590KDDaFtDQqPPEezo= github.com/DataDog/datadog-api-client-go/v2 v2.43.0 h1:HiRaKLfMe9qnoZ1r6Gm0dHg30i9wi1YqdYPVJ8/Fkgc= diff --git a/exporter/clickhouseexporter/go.mod b/exporter/clickhouseexporter/go.mod index 9ce5712eb7ead..809cb9433b8b8 100644 --- a/exporter/clickhouseexporter/go.mod +++ b/exporter/clickhouseexporter/go.mod @@ -37,7 +37,7 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v28.3.2+incompatible // indirect + github.com/docker/docker v28.3.3+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.8.4 // indirect diff --git a/exporter/clickhouseexporter/go.sum b/exporter/clickhouseexporter/go.sum index 7bc45d0b98f13..d3347db0c88a4 100644 --- a/exporter/clickhouseexporter/go.sum +++ b/exporter/clickhouseexporter/go.sum @@ -33,8 +33,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.2+incompatible h1:wn66NJ6pWB1vBZIilP8G3qQPqHy5XymfYn5vsqeA5oA= -github.com/docker/docker v28.3.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= diff --git a/exporter/datadogexporter/go.mod b/exporter/datadogexporter/go.mod index 2ea83df130d17..e4c4f542c1feb 100644 --- a/exporter/datadogexporter/go.mod +++ b/exporter/datadogexporter/go.mod @@ -149,7 +149,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/statstracker v0.66.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.67.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 // indirect github.com/DataDog/datadog-agent/pkg/version v0.68.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect github.com/DataDog/go-sqllexer v0.1.6 // indirect diff --git a/exporter/datadogexporter/go.sum b/exporter/datadogexporter/go.sum index b6a3fb0190d90..ad86d06c2b706 100644 --- a/exporter/datadogexporter/go.sum +++ b/exporter/datadogexporter/go.sum @@ -220,8 +220,8 @@ github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 h1:pM5p51BfBYgkp github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0/go.mod h1:cQWaDB4GxK3i+0jbXZlePXzJ3f1oz2bnEPVfF9rK1Dg= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0 h1:mLXT+D29nRHZdUHQvLp+mskZ9kQZJqkkk05EsEjw7WY= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0/go.mod h1:yi2rztz2PWPdqIykyyX4nHkJiQLvEn0dTPzj9V7g7o8= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 h1:as8nOTMEeifcLCeHN+oqfdRrtUtVZIvwIN7kD2fQDIM= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0/go.mod h1:2l1nTFRygeIQkYr8E1tHuO/ZMjxPt0VgJ1wW14F2Fb4= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 h1:8euFCQn/SNpuN3L5uHx0bHj3N6Yf4QBukUSdylwArSA= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608/go.mod h1:mKV8FnTNGRSJF/TEPDOYWCCnarpPsxhfTZBVkFh/zRY= github.com/DataDog/datadog-agent/pkg/version v0.68.0 h1:O9K/SxuDrKJp+juA724SS1STQxw7meylHWTHjlHs+rM= github.com/DataDog/datadog-agent/pkg/version v0.68.0/go.mod h1:zpRXbtsHTgxP3vMyL6hsKQt/c590KDDaFtDQqPPEezo= github.com/DataDog/datadog-api-client-go/v2 v2.43.0 h1:HiRaKLfMe9qnoZ1r6Gm0dHg30i9wi1YqdYPVJ8/Fkgc= diff --git a/exporter/datadogexporter/integrationtest/go.mod b/exporter/datadogexporter/integrationtest/go.mod index 89111e47d139b..346ccfa7b9177 100644 --- a/exporter/datadogexporter/integrationtest/go.mod +++ b/exporter/datadogexporter/integrationtest/go.mod @@ -123,7 +123,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/statstracker v0.66.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.67.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 // indirect github.com/DataDog/datadog-agent/pkg/version v0.68.0 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.43.0 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect diff --git a/exporter/datadogexporter/integrationtest/go.sum b/exporter/datadogexporter/integrationtest/go.sum index ff4afe27e5438..ae59c929aae96 100644 --- a/exporter/datadogexporter/integrationtest/go.sum +++ b/exporter/datadogexporter/integrationtest/go.sum @@ -213,8 +213,8 @@ github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 h1:pM5p51BfBYgkp github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0/go.mod h1:cQWaDB4GxK3i+0jbXZlePXzJ3f1oz2bnEPVfF9rK1Dg= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0 h1:mLXT+D29nRHZdUHQvLp+mskZ9kQZJqkkk05EsEjw7WY= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0/go.mod h1:yi2rztz2PWPdqIykyyX4nHkJiQLvEn0dTPzj9V7g7o8= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 h1:as8nOTMEeifcLCeHN+oqfdRrtUtVZIvwIN7kD2fQDIM= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0/go.mod h1:2l1nTFRygeIQkYr8E1tHuO/ZMjxPt0VgJ1wW14F2Fb4= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 h1:8euFCQn/SNpuN3L5uHx0bHj3N6Yf4QBukUSdylwArSA= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608/go.mod h1:mKV8FnTNGRSJF/TEPDOYWCCnarpPsxhfTZBVkFh/zRY= github.com/DataDog/datadog-agent/pkg/version v0.68.0 h1:O9K/SxuDrKJp+juA724SS1STQxw7meylHWTHjlHs+rM= github.com/DataDog/datadog-agent/pkg/version v0.68.0/go.mod h1:zpRXbtsHTgxP3vMyL6hsKQt/c590KDDaFtDQqPPEezo= github.com/DataDog/datadog-api-client-go/v2 v2.43.0 h1:HiRaKLfMe9qnoZ1r6Gm0dHg30i9wi1YqdYPVJ8/Fkgc= diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index dccbb7b6e98d0..f2de698548ffd 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -82,14 +82,45 @@ As a consequence of supporting [confighttp], the Elasticsearch exporter also sup The Elasticsearch exporter sets `timeout` (HTTP request timeout) to 90s by default. All other defaults are as defined by [confighttp]. -### Queuing +### Queuing and batching -The Elasticsearch exporter supports the common [`sending_queue` settings][exporterhelper]. However, the sending queue is currently disabled by default. +The exporter is transitioning from its own internal batching to OpenTelemetry's standard +queueing and batching. The below sections describe the current default and the latest +configuration option for queueing and batching available via the `sending_queue` configuration. -### Batching +#### Internal batching by Elasticsearch exporter + +By default, the exporter will perform its own buffering and batching, as configured through the +`flush` config. In this case both `sending_queue` and `batcher` will be unused. The exporter +will perform its own buffering and batching and will issue async requests to Elasticsearch in +all cases other than if any of the following conditions are met: + +- `sending_queue::batch` is defined (irrespective of `sending_queue` being enabled or not) +- `batcher::enabled` is defined (set to `true` or `false`) + +In a future release when the `sending_queue` config is stable, and has feature parity +with the exporter's existing `flush` config, it will be enabled by default. + +Using the `sending_queue` functionality provides several benefits over the default behavior: + - With a persistent queue, or no queue at all, `sending_queue` enables at least once delivery. + On the other hand, with the default behavior, the exporter will accept data and process it + asynchronously, which interacts poorly with queueing. + - By ensuring the exporter makes requests to Elasticsearch synchronously (batching disabled), + client metadata can be passed through to Elasticsearch requests, + e.g. by using the [`headers_setter` extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/extension/headerssetterextension/README.md). + +#### Queueing and batching using sending queue + +The Elasticsearch exporter supports the common [`sending_queue` settings][exporterhelper] which +supports both queueing and batching. However, the sending queue is currently disabled by +default. Sending queue can be enabled by setting `sending_queue::enabled` to `true`. The batching support in sending queue is also disabled by default. Batching can be enabled by defining `sending_queue::batch`. + +The [`exporterhelper` documentation][exporterhelper] provides more details on the `sending_queue` settings. + +#### Deprecated batcher config > [!WARNING] -> The `batcher` config is experimental and may change without notice. +> The `batcher` config is now deprecated and will be removed in an upcoming version. Check the [queueing and batching](#queueing-and-batching) section for using the `sending_queue` setting that supersedes `batcher`. In the interim, `batcher` configurations are still valid, however, they will be ignored if `sending_queue::batch` is defined even if `sending_queue` is not enabled. The Elasticsearch exporter supports the [common `batcher` settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/internal/queue_sender.go). @@ -100,21 +131,6 @@ The Elasticsearch exporter supports the [common `batcher` settings](https://gith - `max_size` (default=0): Maximum batch size to be exported to Elasticsearch, measured in units according to `batcher::sizer`. To limit bulk request size, configure `flush::bytes` instead. :warning: It is recommended to keep `max_size` as 0 as a non-zero value may lead to broken metrics grouping and indexing rejections. - `flush_timeout` (default=30s): Maximum time of the oldest item spent inside the batcher buffer, aka "max age of batcher buffer". A batcher flush will happen regardless of the size of content in batcher buffer. -By default, the exporter will perform its own buffering and batching, as configured through the -`flush` config, and `batcher` will be unused. By setting `batcher::enabled` to either `true` or -`false`, the exporter will not perform any of its own buffering or batching, and the `flush::interval` config -will be ignored. -In a future release when the `batcher` config is stable, and has feature parity -with the exporter's existing `flush` config, it will be enabled by default. - -Using the common `batcher` functionality provides several benefits over the default behavior: - - Combined with a persistent queue, or no queue at all, `batcher` enables at least once delivery. - With the default behavior, the exporter will accept data and process it asynchronously, - which interacts poorly with queuing. - - By ensuring the exporter makes requests to Elasticsearch synchronously, - client metadata can be passed through to Elasticsearch requests, - e.g. by using the [`headers_setter` extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/extension/headerssetterextension/README.md). - ### Elasticsearch document routing Documents are statically or dynamically routed to the target index / data stream in the following order. The first routing mode that applies will be used. @@ -359,11 +375,18 @@ The Elasticsearch Exporter's own telemetry settings for testing and debugging pu ### Metadata keys -Metadata keys are a list of client metadata keys that the exporter currently uses to enrich internal telemetry. +Metadata keys are a list of client metadata keys that the exporter uses to partition batches +when `sending_queue` is enabled with batching support and enrich internal telemetry. ⚠️ This is experimental and may change at any time. -- `metadata_keys` (optional): List of metadata keys that will be added to the exporter's telemetry if defined. The config only applies when batcher is used (set to `true` or `false`). The metadata keys are converted to lower case as key lookups for client metadata is case insensitive. This means that the metric produced by internal telemetry will also have the attribute in lower case. +- `metadata_keys` (optional): List of metadata keys that will be used to partition the data +into batches if [sending_queue][exporterhelper] is enabled with batching support OR +`batcher::enabled` is set. The keys will also be used to enrich the exporter's internal +telemetry if defined. The keys are extracted from the client metadata available via the context +and added to the internal telemetry as attributes. + +NOTE: The metadata keys are converted to lower case as key lookups for client metadata is case insensitive. This means that the metric produced by internal telemetry will also have the attribute in lower case. ## Exporting metrics diff --git a/exporter/elasticsearchexporter/bulkindexer.go b/exporter/elasticsearchexporter/bulkindexer.go index 64ae989531192..ddf79d218d29a 100644 --- a/exporter/elasticsearchexporter/bulkindexer.go +++ b/exporter/elasticsearchexporter/bulkindexer.go @@ -71,7 +71,7 @@ func newBulkIndexer( tb *metadata.TelemetryBuilder, logger *zap.Logger, ) (bulkIndexer, error) { - if config.Batcher.enabledSet { + if config.Batcher.enabledSet || (config.QueueBatchConfig.Enabled && config.QueueBatchConfig.Batch.HasValue()) { return newSyncBulkIndexer(client, config, requireDataStream, tb, logger), nil } return newAsyncBulkIndexer(client, config, requireDataStream, tb, logger) diff --git a/exporter/elasticsearchexporter/bulkindexer_test.go b/exporter/elasticsearchexporter/bulkindexer_test.go index 79624ff44d93e..cd01c01ca5ff3 100644 --- a/exporter/elasticsearchexporter/bulkindexer_test.go +++ b/exporter/elasticsearchexporter/bulkindexer_test.go @@ -21,6 +21,7 @@ import ( "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config/configcompression" "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/sdk/metric/metricdata" "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" @@ -664,3 +665,131 @@ func TestSyncBulkIndexer_flushBytes(t *testing.T) { }, }, metricdatatest.IgnoreTimestamp()) } + +func TestNewBulkIndexer(t *testing.T) { + for _, tc := range []struct { + name string + config map[string]any + expectSyncBulkIndexer bool + }{ + { + name: "batcher_enabled_unset", + config: map[string]any{ + "batcher": map[string]any{ + "min_size": 100, + "max_size": 200, + }, + }, + expectSyncBulkIndexer: false, + }, + { + name: "batcher_enabled=true", + config: map[string]any{ + "batcher": map[string]any{ + "enabled": true, + "min_size": 100, + "max_size": 200, + }, + }, + expectSyncBulkIndexer: true, + }, + { + name: "batcher_enabled=true", + config: map[string]any{ + "batcher": map[string]any{ + "enabled": false, + "min_size": 100, + "max_size": 200, + }, + }, + expectSyncBulkIndexer: true, + }, + { + name: "sending_queue_enabled_without_batcher", + config: map[string]any{ + "sending_queue": map[string]any{ + "enabled": true, + }, + }, + expectSyncBulkIndexer: false, + }, + { + name: "sending_queue__with_batch_enabled", + config: map[string]any{ + "sending_queue": map[string]any{ + "enabled": true, + "batch": map[string]any{ + "min_size": 100, + "max_size": 200, + }, + }, + }, + expectSyncBulkIndexer: true, + }, + { + name: "sending_queue_disabled_but_batch_configured", + config: map[string]any{ + "sending_queue": map[string]any{ + "enabled": false, + "batch": map[string]any{ + "min_size": 100, + "max_size": 200, + }, + }, + "batcher": map[string]any{ + // no enabled set + "min_size": 100, + "max_size": 200, + }, + }, + expectSyncBulkIndexer: false, + }, + { + name: "sending_queue_overrides_batcher", + config: map[string]any{ + "sending_queue": map[string]any{ + "enabled": true, + "batch": map[string]any{ + "min_size": 100, + "max_size": 200, + }, + }, + "batcher": map[string]any{ + "enabled": true, + "min_size": 100, + "max_size": 200, + }, + }, + expectSyncBulkIndexer: true, + }, + { + name: "sending_queue_without_batch_with_batcher_enabled", + config: map[string]any{ + "sending_queue": map[string]any{ + "enabled": true, + }, + "batcher": map[string]any{ + "enabled": true, + "min_size": 100, + "max_size": 200, + }, + }, + expectSyncBulkIndexer: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + client, err := elasticsearch.NewDefaultClient() + require.NoError(t, err) + cfg := createDefaultConfig() + cm := confmap.NewFromStringMap(tc.config) + require.NoError(t, cm.Unmarshal(cfg)) + + bi, err := newBulkIndexer(client, cfg.(*Config), true, nil, nil) + require.NoError(t, err) + t.Cleanup(func() { bi.Close(context.Background()) }) + + _, ok := bi.(*syncBulkIndexer) + assert.Equal(t, tc.expectSyncBulkIndexer, ok) + }) + } +} diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 7cfaadcb903d6..b1ba346b0097e 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -23,7 +23,11 @@ import ( // Config defines configuration for Elastic exporter. type Config struct { - QueueSettings exporterhelper.QueueBatchConfig `mapstructure:"sending_queue"` + // QueueBatchConfig configures the sending queue and the batching done + // by the exporter. The performed batching can further be customized by + // configuring `metadata_keys` which will be used to partition the batches. + QueueBatchConfig exporterhelper.QueueBatchConfig `mapstructure:"sending_queue"` + // Endpoints holds the Elasticsearch URLs the exporter should send events to. // // This setting is required if CloudID is not set and if the @@ -99,15 +103,20 @@ type Config struct { // Batcher is unused by default, in which case Flush will be used. // If Batcher.Enabled is non-nil (i.e. batcher::enabled is specified), // then the Flush will be ignored even if Batcher.Enabled is false. - // TODO: Deprecate and remove this section in favor of sending_queue::batch. + // + // Deprecated: [v0.132.0] This config is now deprecated. Use `sending_queue::batch` instead. + // Batcher config will be ignored if `sending_queue::batch` is defined even if sending queue + // is disabled. Batcher BatcherConfig `mapstructure:"batcher"` // Experimental: MetadataKeys defines a list of client.Metadata keys that - // will be added to the exporter's telemetry if defined. The config only - // applies when batcher is used (set to `true` or `false`). The metadata keys - // are converted to lower case as key lookups for client metadata is case - // insensitive. This means that the metric produced by internal telemetry - // will also have the attribute in lower case. + // will be used as partition keys for when batcher is enabled and will be + // added to the exporter's telemetry if defined. The config only applies + // when `sending_queue::batch` is defined or when the, now deprecated, batcher + // is used (set to `true` or `false`). The metadata keys are converted to + // lower case as key lookups for client metadata is case insensitive. This + // means that the metric produced by internal telemetry will also have the + // attribute in lower case. // // Keys are case-insensitive and duplicates will trigger a validation error. MetadataKeys []string `mapstructure:"metadata_keys"` @@ -493,6 +502,12 @@ func handleDeprecatedConfig(cfg *Config, logger *zap.Logger) { if cfg.TracesDynamicIndex.Enabled { logger.Warn("traces_dynamic_index::enabled has been deprecated, and will be removed in a future version. It is now a no-op. Dynamic document routing is now the default. See Elasticsearch Exporter README.") } + switch { + case cfg.Batcher.enabledSet && cfg.QueueBatchConfig.Batch.HasValue(): + logger.Warn("batcher::enabled and sending_queue::batch both have been set, sending_queue::batch will take preference.") + case cfg.Batcher.enabledSet: + logger.Warn("batcher has been deprecated, and will be removed in a future version. Use sending_queue instead.") + } } func handleTelemetryConfig(cfg *Config, logger *zap.Logger) { diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index 2ce4d3edebb91..fb1409b1bf9b2 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -17,6 +17,7 @@ import ( "go.opentelemetry.io/collector/config/configcompression" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configoptional" "go.opentelemetry.io/collector/confmap/confmaptest" "go.opentelemetry.io/collector/confmap/xconfmap" "go.opentelemetry.io/collector/exporter/exporterhelper" @@ -56,7 +57,7 @@ func TestConfig(t *testing.T) { id: component.NewIDWithName(metadata.Type, "trace"), configFile: "config.yaml", expected: &Config{ - QueueSettings: exporterhelper.QueueBatchConfig{ + QueueBatchConfig: exporterhelper.QueueBatchConfig{ Enabled: false, NumConsumers: exporterhelper.NewDefaultQueueConfig().NumConsumers, QueueSize: exporterhelper.NewDefaultQueueConfig().QueueSize, @@ -132,7 +133,7 @@ func TestConfig(t *testing.T) { id: component.NewIDWithName(metadata.Type, "log"), configFile: "config.yaml", expected: &Config{ - QueueSettings: exporterhelper.QueueBatchConfig{ + QueueBatchConfig: exporterhelper.QueueBatchConfig{ Enabled: true, NumConsumers: exporterhelper.NewDefaultQueueConfig().NumConsumers, QueueSize: exporterhelper.NewDefaultQueueConfig().QueueSize, @@ -208,7 +209,7 @@ func TestConfig(t *testing.T) { id: component.NewIDWithName(metadata.Type, "metric"), configFile: "config.yaml", expected: &Config{ - QueueSettings: exporterhelper.QueueBatchConfig{ + QueueBatchConfig: exporterhelper.QueueBatchConfig{ Enabled: true, NumConsumers: exporterhelper.NewDefaultQueueConfig().NumConsumers, QueueSize: exporterhelper.NewDefaultQueueConfig().QueueSize, @@ -360,6 +361,22 @@ func TestConfig(t *testing.T) { cfg.MetadataKeys = []string{"x-test-1", "x-test-2"} }), }, + { + id: component.NewIDWithName(metadata.Type, "queuebatch_enabled"), + configFile: "config.yaml", + expected: withDefaultConfig(func(cfg *Config) { + cfg.Endpoint = "https://elastic.example.com:9200" + + cfg.QueueBatchConfig.Enabled = true + cfg.QueueBatchConfig.NumConsumers = 100 + cfg.QueueBatchConfig.Sizer = exporterhelper.RequestSizerTypeRequests + cfg.QueueBatchConfig.Batch = configoptional.Some( + exporterhelper.BatchConfig{ + Sizer: exporterhelper.RequestSizerTypeRequests, + }, + ) + }), + }, } for _, tt := range tests { diff --git a/exporter/elasticsearchexporter/exporter.go b/exporter/elasticsearchexporter/exporter.go index a79b3d779e504..26beaae62c197 100644 --- a/exporter/elasticsearchexporter/exporter.go +++ b/exporter/elasticsearchexporter/exporter.go @@ -352,6 +352,8 @@ func (e *elasticsearchExporter) pushTraceData( ctx context.Context, td ptrace.Traces, ) error { + // Get the partioner key from the context + // Decode the key to get the info defaultMappingMode, err := e.getRequestMappingMode(ctx) if err != nil { return err diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index af9bbcef325f6..9255c05b8b0ae 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -50,8 +50,8 @@ func createDefaultConfig() component.Config { httpClientConfig.CompressionParams.Level = gzip.BestSpeed return &Config{ - QueueSettings: qs, - ClientConfig: httpClientConfig, + QueueBatchConfig: qs, + ClientConfig: httpClientConfig, LogsDynamicID: DynamicIDSettings{ Enabled: false, }, @@ -113,12 +113,17 @@ func createLogsExporter( return nil, err } + qbs := exporterhelper.NewLogsQueueBatchSettings() + if len(cf.MetadataKeys) > 0 { + qbs.Partitioner = metadataKeysPartitioner{keys: cf.MetadataKeys} + } + return exporterhelper.NewLogs( ctx, set, cfg, exporter.pushLogsData, - exporterhelperOptions(cf, exporter.Start, exporter.Shutdown)..., + exporterhelperOptions(cf, exporter.Start, exporter.Shutdown, qbs)..., ) } @@ -136,12 +141,17 @@ func createMetricsExporter( return nil, err } + qbs := exporterhelper.NewMetricsQueueBatchSettings() + if len(cf.MetadataKeys) > 0 { + qbs.Partitioner = metadataKeysPartitioner{keys: cf.MetadataKeys} + } + return exporterhelper.NewMetrics( ctx, set, cfg, exporter.pushMetricsData, - exporterhelperOptions(cf, exporter.Start, exporter.Shutdown)..., + exporterhelperOptions(cf, exporter.Start, exporter.Shutdown, qbs)..., ) } @@ -158,12 +168,17 @@ func createTracesExporter(ctx context.Context, return nil, err } + qbs := exporterhelper.NewTracesQueueBatchSettings() + if len(cf.MetadataKeys) > 0 { + qbs.Partitioner = metadataKeysPartitioner{keys: cf.MetadataKeys} + } + return exporterhelper.NewTraces( ctx, set, cfg, exporter.pushTraceData, - exporterhelperOptions(cf, exporter.Start, exporter.Shutdown)..., + exporterhelperOptions(cf, exporter.Start, exporter.Shutdown, qbs)..., ) } @@ -185,12 +200,17 @@ func createProfilesExporter( return nil, err } + qbs := xexporterhelper.NewProfilesQueueBatchSettings() + if len(cf.MetadataKeys) > 0 { + qbs.Partitioner = metadataKeysPartitioner{keys: cf.MetadataKeys} + } + return xexporterhelper.NewProfiles( ctx, set, cfg, exporter.pushProfilesData, - exporterhelperOptions(cf, exporter.Start, exporter.Shutdown)..., + exporterhelperOptions(cf, exporter.Start, exporter.Shutdown, qbs)..., ) } @@ -198,16 +218,30 @@ func exporterhelperOptions( cfg *Config, start component.StartFunc, shutdown component.ShutdownFunc, + qbs exporterhelper.QueueBatchSettings, ) []exporterhelper.Option { opts := []exporterhelper.Option{ exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: false}), exporterhelper.WithStart(start), exporterhelper.WithShutdown(shutdown), } - qs := cfg.QueueSettings - if cfg.Batcher.enabledSet { + qbc := cfg.QueueBatchConfig + switch { + case qbc.Batch.HasValue(): + // Latest queue batch settings are used, prioritize them even if sending queue is disabled + opts = append(opts, exporterhelper.WithQueueBatch(qbc, qbs)) + + // Effectively disable timeout_sender because timeout is enforced in bulk indexer. + // + // We keep timeout_sender enabled in the async mode (sending_queue not enabled OR sending + // queue enabled but batching not enabled OR based on the deprecated batcher setting), to + // ensure sending data to the background workers will not block indefinitely. + if qbc.Enabled { + opts = append(opts, exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0})) + } + case cfg.Batcher.enabledSet: if cfg.Batcher.Enabled { - qs.Batch = configoptional.Some(exporterhelper.BatchConfig{ + qbc.Batch = configoptional.Some(exporterhelper.BatchConfig{ FlushTimeout: cfg.Batcher.FlushTimeout, MinSize: cfg.Batcher.MinSize, MaxSize: cfg.Batcher.MaxSize, @@ -216,18 +250,23 @@ func exporterhelperOptions( // If the deprecated batcher is enabled without a queue, enable blocking queue to replicate the // behavior of the deprecated batcher. - if !qs.Enabled { - qs.Enabled = true - qs.WaitForResult = true + if !qbc.Enabled { + qbc.Enabled = true + qbc.WaitForResult = true } } - // Effectively disable timeout_sender because timeout is enforced in bulk indexer. - // - // We keep timeout_sender enabled in the async mode (Batcher.Enabled == nil), - // to ensure sending data to the background workers will not block indefinitely. - opts = append(opts, exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0})) + opts = append( + opts, + // Effectively disable timeout_sender because timeout is enforced in bulk indexer. + // + // We keep timeout_sender enabled in the async mode (Batcher.Enabled == nil), + // to ensure sending data to the background workers will not block indefinitely. + exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0}), + exporterhelper.WithQueue(qbc), + ) + default: + opts = append(opts, exporterhelper.WithQueue(qbc)) } - opts = append(opts, exporterhelper.WithQueue(qs)) return opts } diff --git a/exporter/elasticsearchexporter/factory_test.go b/exporter/elasticsearchexporter/factory_test.go index 089b1b4de3a75..879a7cc09f694 100644 --- a/exporter/elasticsearchexporter/factory_test.go +++ b/exporter/elasticsearchexporter/factory_test.go @@ -10,7 +10,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/exporter/exportertest" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter/internal/metadata" ) @@ -23,40 +26,144 @@ func TestCreateDefaultConfig(t *testing.T) { } func TestFactory_CreateLogs(t *testing.T) { - factory := NewFactory() - cfg := withDefaultConfig(func(cfg *Config) { - cfg.Endpoints = []string{"http://test:9200"} - }) - params := exportertest.NewNopSettings(metadata.Type) - exporter, err := factory.CreateLogs(context.Background(), params, cfg) - require.NoError(t, err) - require.NotNil(t, exporter) - - require.NoError(t, exporter.Shutdown(context.Background())) + for _, tc := range signalTestCases { + t.Run(tc.name, func(t *testing.T) { + factory := NewFactory() + params := exportertest.NewNopSettings(metadata.Type) + + observedZapCore, observedLogs := observer.New(zap.InfoLevel) + params.Logger = zap.New(observedZapCore) + + cfg := createDefaultConfig() + cm := confmap.NewFromStringMap(tc.cfg) + require.NoError(t, cm.Unmarshal(cfg)) + + exporter, err := factory.CreateLogs(context.Background(), params, cfg.(*Config)) + require.NoError(t, err) + require.NotNil(t, exporter) + require.Equal(t, len(tc.expectedLogs), observedLogs.Len()) + actualLogs := observedLogs.All() + for i, expectedLog := range tc.expectedLogs { + assert.Contains(t, actualLogs[i].Message, expectedLog) + } + + require.NoError(t, exporter.Shutdown(context.Background())) + }) + } } func TestFactory_CreateMetrics(t *testing.T) { - factory := NewFactory() - cfg := withDefaultConfig(func(cfg *Config) { - cfg.Endpoints = []string{"http://test:9200"} - }) - params := exportertest.NewNopSettings(metadata.Type) - exporter, err := factory.CreateMetrics(context.Background(), params, cfg) - require.NoError(t, err) - require.NotNil(t, exporter) - - require.NoError(t, exporter.Shutdown(context.Background())) + for _, tc := range signalTestCases { + t.Run(tc.name, func(t *testing.T) { + factory := NewFactory() + params := exportertest.NewNopSettings(metadata.Type) + + observedZapCore, observedLogs := observer.New(zap.InfoLevel) + params.Logger = zap.New(observedZapCore) + + cfg := createDefaultConfig() + cm := confmap.NewFromStringMap(tc.cfg) + require.NoError(t, cm.Unmarshal(cfg)) + + exporter, err := factory.CreateMetrics(context.Background(), params, cfg.(*Config)) + require.NoError(t, err) + require.NotNil(t, exporter) + require.Equal(t, len(tc.expectedLogs), observedLogs.Len()) + actualLogs := observedLogs.All() + for i, expectedLog := range tc.expectedLogs { + assert.Contains(t, actualLogs[i].Message, expectedLog) + } + + require.NoError(t, exporter.Shutdown(context.Background())) + }) + } } func TestFactory_CreateTraces(t *testing.T) { - factory := NewFactory() - cfg := withDefaultConfig(func(cfg *Config) { - cfg.Endpoints = []string{"http://test:9200"} - }) - params := exportertest.NewNopSettings(metadata.Type) - exporter, err := factory.CreateTraces(context.Background(), params, cfg) - require.NoError(t, err) - require.NotNil(t, exporter) - - require.NoError(t, exporter.Shutdown(context.Background())) + for _, tc := range signalTestCases { + t.Run(tc.name, func(t *testing.T) { + factory := NewFactory() + params := exportertest.NewNopSettings(metadata.Type) + + observedZapCore, observedLogs := observer.New(zap.InfoLevel) + params.Logger = zap.New(observedZapCore) + + cfg := createDefaultConfig() + cm := confmap.NewFromStringMap(tc.cfg) + require.NoError(t, cm.Unmarshal(cfg)) + + exporter, err := factory.CreateTraces(context.Background(), params, cfg.(*Config)) + require.NoError(t, err) + require.NotNil(t, exporter) + require.Equal(t, len(tc.expectedLogs), observedLogs.Len()) + actualLogs := observedLogs.All() + for i, expectedLog := range tc.expectedLogs { + assert.Contains(t, actualLogs[i].Message, expectedLog) + } + + require.NoError(t, exporter.Shutdown(context.Background())) + }) + } +} + +var signalTestCases = []struct { + name string + cfg map[string]any + expectedLogs []string +}{ + { + name: "default", + cfg: map[string]any{ + "endpoints": []string{"http://test:9200"}, + }, + }, + { + name: "with_deprecated_batcher", + cfg: map[string]any{ + "batcher": map[string]any{ + "enabled": true, + }, + }, + expectedLogs: []string{ + "batcher has been deprecated", + }, + }, + { + name: "with_sending_queue", + cfg: map[string]any{ + "sending_queue": map[string]any{ + "enabled": true, + "batch": map[string]any{}, + }, + }, + }, + { + name: "with_sending_queue_disabled_and_deprecated_batcher", + cfg: map[string]any{ + "sending_queue": map[string]any{ + "batch": map[string]any{}, + }, + "batcher": map[string]any{ + "enabled": true, + }, + }, + expectedLogs: []string{ + "sending_queue::batch will take preference", + }, + }, + { + name: "with_sending_queue_enabled_and_deprecated_batcher", + cfg: map[string]any{ + "sending_queue": map[string]any{ + "enabled": true, + "batch": map[string]any{}, + }, + "batcher": map[string]any{ + "enabled": true, + }, + }, + expectedLogs: []string{ + "sending_queue::batch will take preference", + }, + }, } diff --git a/exporter/elasticsearchexporter/partitioner.go b/exporter/elasticsearchexporter/partitioner.go new file mode 100644 index 0000000000000..415ad7baf0332 --- /dev/null +++ b/exporter/elasticsearchexporter/partitioner.go @@ -0,0 +1,40 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package elasticsearchexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter" + +import ( + "bytes" + "context" + + "go.opentelemetry.io/collector/client" + "go.opentelemetry.io/collector/exporter/exporterhelper" +) + +type metadataKeysPartitioner struct { + keys []string +} + +func (p metadataKeysPartitioner) GetKey( + ctx context.Context, + _ exporterhelper.Request, +) string { + var kb bytes.Buffer + meta := client.FromContext(ctx).Metadata + + var afterFirst bool + for _, k := range p.keys { + if values := meta.Get(k); len(values) != 0 { + if afterFirst { + kb.WriteByte(0) + } + kb.WriteString(k) + afterFirst = true + for _, val := range values { + kb.WriteByte(0) + kb.WriteString(val) + } + } + } + return kb.String() +} diff --git a/exporter/elasticsearchexporter/partitioner_test.go b/exporter/elasticsearchexporter/partitioner_test.go new file mode 100644 index 0000000000000..9db5128540fcf --- /dev/null +++ b/exporter/elasticsearchexporter/partitioner_test.go @@ -0,0 +1,78 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package elasticsearchexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter" + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/client" +) + +func TestGetKey(t *testing.T) { + for _, tc := range []struct { + name string + metadataKeys []string + metadata map[string][]string + expected string + }{ + { + name: "empty", + metadataKeys: nil, + metadata: map[string][]string{ + "key1": {"val1"}, + }, + expected: "", + }, + { + name: "with_missing_key", + metadataKeys: []string{"key404"}, + metadata: map[string][]string{ + "key1": {"val1"}, + }, + expected: "", + }, + { + name: "with_key_in_metadata", + metadataKeys: []string{"key1"}, + metadata: map[string][]string{ + "key1": {"val1"}, + }, + expected: "key1\x00val1", + }, + { + name: "with_multiple_key_in_metadata", + metadataKeys: []string{"key1", "key2"}, + metadata: map[string][]string{ + "key1": {"val1"}, + "key2": {"val2.1", "val2.2", "val2.3"}, + }, + expected: "key1\x00val1\x00key2\x00val2.1\x00val2.2\x00val2.3", + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := client.NewContext(context.Background(), client.Info{ + Metadata: client.NewMetadata(tc.metadata), + }) + assert.Equal(t, tc.expected, metadataKeysPartitioner{keys: tc.metadataKeys}.GetKey(ctx, nil)) + }) + } +} + +func BenchmarkGetKey(b *testing.B) { + p := metadataKeysPartitioner{keys: []string{"key1", "key2"}} + ctx := client.NewContext(context.Background(), client.Info{ + Metadata: client.NewMetadata(map[string][]string{ + "key1": {"val1"}, + "key2": {"val2.1", "val2.2", "val2.3"}, + }), + }) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = p.GetKey(ctx, nil) + } +} diff --git a/exporter/elasticsearchexporter/testdata/config.yaml b/exporter/elasticsearchexporter/testdata/config.yaml index 57eb199e6e7d6..796b3afa5f2ea 100644 --- a/exporter/elasticsearchexporter/testdata/config.yaml +++ b/exporter/elasticsearchexporter/testdata/config.yaml @@ -120,3 +120,9 @@ elasticsearch/metadata_keys: metadata_keys: - x-test-1 - x-test-2 +elasticsearch/queuebatch_enabled: + endpoint: https://elastic.example.com:9200 + sending_queue: + enabled: true + num_consumers: 100 + batch: {} diff --git a/exporter/kafkaexporter/README.md b/exporter/kafkaexporter/README.md index 496745a4d3090..191a2aa643b15 100644 --- a/exporter/kafkaexporter/README.md +++ b/exporter/kafkaexporter/README.md @@ -48,7 +48,7 @@ The following settings can be optionally configured: - `encoding` (default = otlp\_proto): The encoding for traces. See [Supported encodings](#supported-encodings). - `topic_from_metadata_key` (default = ""): The name of the metadata key whose value should be used as the message's topic. Useful to dynamically produce to topics based on request inputs. It takes precedence over `topic_from_attribute` and `topic` settings. - `topic` (Deprecated in v0.124.0: use `logs::topic`, `metrics::topic`, and `traces::topic`) If specified, this is used as the default topic, but will be overridden by signal-specific configuration. See [Destination Topic](#destination-topic) below for more details. -- `topic_from_attribute` (default = ""): Specify the resource attribute whose value should be used as the message's topic. See [Destination Topic](#destination-topic) below for more details. +- `topic_from_attribute` (default = ""): Specify the resource attribute whose value should be used as the message's topic. See [Destination Topic](#destination-topic) below for more details. - `encoding` (Deprecated in v0.124.0: use `logs::encoding`, `metrics::encoding`, and `traces::encoding`) If specified, this is used as the default encoding, but will be overridden by signal-specific configuration. See [Supported encodings](#supported-encodings) below for more details. - `include_metadata_keys` (default = []): Specifies a list of metadata keys to propagate as Kafka message headers. If one or more keys aren't found in the metadata, they are ignored. - `partition_traces_by_id` (default = false): configures the exporter to include the trace ID as the message key in trace messages sent to kafka. *Please note:* this setting does not have any effect on Jaeger encoding exporters since Jaeger exporters include trace ID as the message key by default. @@ -83,7 +83,7 @@ The following settings can be optionally configured: - `retry` - `max` (default = 3): The number of retries to get metadata - `backoff` (default = 250ms): How long to wait between metadata retries -- `timeout` (default = 5s): Is the timeout for every attempt to send data to the backend. +- `timeout` (default = 5s): Time to wait per individual attempt to produce data to Kafka. - `retry_on_failure` - `enabled` (default = true) - `initial_interval` (default = 5s): Time to wait after the first failure before retrying; ignored if `enabled` is `false` @@ -98,8 +98,8 @@ The following settings can be optionally configured: - `requests_per_second` is the average number of requests per seconds. - `producer` - `max_message_bytes` (default = 1000000) the maximum permitted size of a message in bytes - - `required_acks` (default = 1) controls when a message is regarded as transmitted. https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#acks - - `compression` (default = 'none') the compression used when producing messages to kafka. The options are: `none`, `gzip`, `snappy`, `lz4`, and `zstd` https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#compression-type + - `required_acks` (default = 1) controls when a message is regarded as transmitted. + - `compression` (default = 'none') the compression used when producing messages to kafka. The options are: `none`, `gzip`, `snappy`, `lz4`, and `zstd` - `compression_params` - `level` (default = -1) the compression level used when producing messages to kafka. - The following are valid combinations of `compression` and `level` @@ -123,17 +123,20 @@ The following settings can be optionally configured: The Kafka exporter supports encoding extensions, as well as the following built-in encodings. Available for all signals: - - `otlp_proto`: data is encoded as OTLP Protobuf - - `otlp_json`: data is encoded as OTLP JSON + +- `otlp_proto`: data is encoded as OTLP Protobuf +- `otlp_json`: data is encoded as OTLP JSON Available only for traces: - - `jaeger_proto`: the payload is serialized to a single Jaeger proto `Span`, and keyed by TraceID. - - `jaeger_json`: the payload is serialized to a single Jaeger JSON Span using `jsonpb`, and keyed by TraceID. - - `zipkin_proto`: the payload is serialized to Zipkin v2 proto Span. - - `zipkin_json`: the payload is serialized to Zipkin v2 JSON Span. + +- `jaeger_proto`: the payload is serialized to a single Jaeger proto `Span`, and keyed by TraceID. +- `jaeger_json`: the payload is serialized to a single Jaeger JSON Span using `jsonpb`, and keyed by TraceID. +- `zipkin_proto`: the payload is serialized to Zipkin v2 proto Span. +- `zipkin_json`: the payload is serialized to Zipkin v2 JSON Span. Available only for logs: - - `raw`: if the log record body is a byte array, it is sent as is. Otherwise, it is serialized to JSON. Resource and record attributes are discarded. + +- `raw`: if the log record body is a byte array, it is sent as is. Otherwise, it is serialized to JSON. Resource and record attributes are discarded. ### Example configuration diff --git a/exporter/prometheusexporter/README.md b/exporter/prometheusexporter/README.md index c2303832f4680..5e26aeabc225e 100644 --- a/exporter/prometheusexporter/README.md +++ b/exporter/prometheusexporter/README.md @@ -31,7 +31,12 @@ The following settings can be optionally configured: - `resource_to_telemetry_conversion` - `enabled` (default = false): If `enabled` is `true`, all the resource attributes will be converted to metric labels by default. - `enable_open_metrics`: (default = `false`): If true, metrics will be exported using the OpenMetrics format. Exemplars are only exported in the OpenMetrics format, and only for histogram and monotonic sum (i.e. counter) metrics. -- `add_metric_suffixes`: (default = `true`): If false, addition of type and unit suffixes is disabled. +- `add_metric_suffixes`: (default = `true`): If false, addition of type and unit suffixes is disabled. **Deprecated**: Use `translation_strategy` instead. This setting is ignored when `translation_strategy` is explicitly set. +- `translation_strategy`: Controls how OTLP metric and attribute names are translated into Prometheus metric and label names. When set, this takes precedence over `add_metric_suffixes`. Available options: + - `UnderscoreEscapingWithSuffixes`: Fully escapes metric names for classic Prometheus metric name compatibility, and includes appending type and unit suffixes. + - `UnderscoreEscapingWithoutSuffixes`: Metric names will continue to escape special characters to `_`, but suffixes won't be attached. + - `NoUTF8EscapingWithSuffixes`: Disables changing special characters to `_`. Special suffixes like units and `_total` for counters will be attached. + - `NoTranslation`: Bypasses all metric and label name translation, passing them through unaltered. Example: @@ -50,7 +55,9 @@ exporters: send_timestamps: true metric_expiration: 180m enable_open_metrics: true + # Legacy configuration - deprecated, ignored when translation_strategy is set add_metric_suffixes: false + translation_strategy: "UnderscoreEscapingWithoutSuffixes" resource_to_telemetry_conversion: enabled: true ``` @@ -59,7 +66,9 @@ Given the example, metrics will be available at `https://1.2.3.4:1234/metrics`. ## Metric names and labels normalization -OpenTelemetry metric names and attributes are normalized to be compliant with Prometheus naming rules. [Details on this normalization process are described in the Prometheus translator module](../../pkg/translator/prometheus/). +By Default, OpenTelemetry metric names and attributes are normalized to be compliant with [Prometheus naming rules](https://prometheus.io/docs/practices/naming/). + +Optionally, users can set different `translation_strategy` options to control how metrics are exposed. Please be aware that Prometheus itself uses content negotiation to decide how to ingest metrics, and underscore escaping might be applied even though this exporter is configured to keep UTF-8 characters. For more details, read [Prometheus' Content Negotiation documentation](https://prometheus.io/docs/instrumenting/content_negotiation/). ## Setting resource attributes as metric labels diff --git a/exporter/prometheusexporter/collector.go b/exporter/prometheusexporter/collector.go index 9bfe3dbe9f494..741454bbc8cbe 100644 --- a/exporter/prometheusexporter/collector.go +++ b/exporter/prometheusexporter/collector.go @@ -31,12 +31,11 @@ type collector struct { accumulator accumulator logger *zap.Logger - sendTimestamps bool - addMetricSuffixes bool - namespace string - constLabels prometheus.Labels - metricFamilies sync.Map - metricExpiration time.Duration + sendTimestamps bool + namespace string + constLabels prometheus.Labels + metricFamilies sync.Map + metricExpiration time.Duration metricNamer otlptranslator.MetricNamer labelNamer otlptranslator.LabelNamer @@ -48,20 +47,67 @@ type metricFamily struct { } func newCollector(config *Config, logger *zap.Logger) *collector { - labelNamer := otlptranslator.LabelNamer{} + labelNamer := configureLabelNamer(config) return &collector{ - accumulator: newAccumulator(logger, config.MetricExpiration), - logger: logger, - namespace: labelNamer.Build(config.Namespace), - sendTimestamps: config.SendTimestamps, - constLabels: config.ConstLabels, - addMetricSuffixes: config.AddMetricSuffixes, - metricExpiration: config.MetricExpiration, - metricNamer: otlptranslator.MetricNamer{WithMetricSuffixes: config.AddMetricSuffixes, Namespace: config.Namespace}, - labelNamer: labelNamer, + accumulator: newAccumulator(logger, config.MetricExpiration), + logger: logger, + namespace: labelNamer.Build(config.Namespace), + sendTimestamps: config.SendTimestamps, + constLabels: config.ConstLabels, + metricExpiration: config.MetricExpiration, + metricNamer: configureMetricNamer(config), + labelNamer: labelNamer, } } +// configureMetricNamer configures the MetricNamer based on the translation strategy or legacy configuration +func configureMetricNamer(config *Config) otlptranslator.MetricNamer { + withSuffixes, utf8Allowed := getTranslationConfiguration(config) + return otlptranslator.MetricNamer{ + WithMetricSuffixes: withSuffixes, + Namespace: config.Namespace, + UTF8Allowed: utf8Allowed, + } +} + +// configureLabelNamer configures the LabelNamer based on the translation strategy or legacy configuration +func configureLabelNamer(config *Config) otlptranslator.LabelNamer { + _, utf8Allowed := getTranslationConfiguration(config) + return otlptranslator.LabelNamer{ + UTF8Allowed: utf8Allowed, + } +} + +// getTranslationConfiguration returns the translation configuration based on the strategy or legacy settings +// Returns (withSuffixes, allowUTF8) +func getTranslationConfiguration(config *Config) (bool, bool) { + // If TranslationStrategy is explicitly set, use it (takes precedence) + if config.TranslationStrategy != "" { + switch config.TranslationStrategy { + case underscoreEscapingWithSuffixes: + return true, false + case underscoreEscapingWithoutSuffixes: + return false, false + case noUTF8EscapingWithSuffixes: + return true, true + case noTranslation: + return false, true + default: + // Fallback to default behavior, suffixes enabled, UTF-8 escaped to underscores. + return true, false + } + } + + // If feature gate is enabled, ignore AddMetricSuffixes (for deprecation) + if disableAddMetricSuffixesFeatureGate.IsEnabled() { + // Default to UnderscoreEscapingWithSuffixes behavior when AddMetricSuffixes is deprecated + return true, false + } + + // Fall back to legacy AddMetricSuffixes behavior, UTF-8 escaped to underscores. + return config.AddMetricSuffixes, false +} + func convertExemplars(exemplars pmetric.ExemplarSlice) []prometheus.Exemplar { length := exemplars.Len() result := make([]prometheus.Exemplar, length) diff --git a/exporter/prometheusexporter/config.go b/exporter/prometheusexporter/config.go index 61e3f77514e9c..7019e2f737195 100644 --- a/exporter/prometheusexporter/config.go +++ b/exporter/prometheusexporter/config.go @@ -4,11 +4,13 @@ package prometheusexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter" import ( + "fmt" "time" "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/featuregate" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry" ) @@ -36,12 +38,49 @@ type Config struct { EnableOpenMetrics bool `mapstructure:"enable_open_metrics"` // AddMetricSuffixes controls whether suffixes are added to metric names. Defaults to true. + // Deprecated: Use TranslationStrategy instead. This setting is ignored when TranslationStrategy is explicitly set. AddMetricSuffixes bool `mapstructure:"add_metric_suffixes"` + + // TranslationStrategy controls how OTLP metric and attribute names are translated into Prometheus metric and label names. + // When set, this takes precedence over AddMetricSuffixes. + TranslationStrategy translationStrategy `mapstructure:"translation_strategy"` } var _ component.Config = (*Config)(nil) // Validate checks if the exporter configuration is valid -func (*Config) Validate() error { +func (cfg *Config) Validate() error { + // Validate translation strategy if set + if cfg.TranslationStrategy != "" { + switch cfg.TranslationStrategy { + case underscoreEscapingWithSuffixes, underscoreEscapingWithoutSuffixes, noUTF8EscapingWithSuffixes, noTranslation: + default: + return fmt.Errorf("invalid translation_strategy: %s", cfg.TranslationStrategy) + } + } return nil } + +type translationStrategy string + +const ( + // underscoreEscapingWithSuffixes fully escapes metric names for classic Prometheus metric name compatibility, + // and includes appending type and unit suffixes + underscoreEscapingWithSuffixes translationStrategy = "UnderscoreEscapingWithSuffixes" + + // underscoreEscapingWithoutSuffixes escapes special characters to '_', but suffixes won't be attached + underscoreEscapingWithoutSuffixes translationStrategy = "UnderscoreEscapingWithoutSuffixes" + + // noUTF8EscapingWithSuffixes disables changing special characters to '_'. Special suffixes like units and '_total' for counters will be attached + noUTF8EscapingWithSuffixes translationStrategy = "NoUTF8EscapingWithSuffixes" + + // noTranslation bypasses all metric and label name translation, passing them through unaltered + noTranslation translationStrategy = "NoTranslation" +) + +var disableAddMetricSuffixesFeatureGate = featuregate.GlobalRegistry().MustRegister( + "exporter.prometheusexporter.DisableAddMetricSuffixes", + featuregate.StageAlpha, + featuregate.WithRegisterDescription("When enabled, the deprecated add_metric_suffixes configuration option is ignored and translation_strategy is always used"), + featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-specification/pull/4533"), +) diff --git a/exporter/prometheusexporter/go.mod b/exporter/prometheusexporter/go.mod index de551dd43ead8..fb3c47ae4a4bd 100644 --- a/exporter/prometheusexporter/go.mod +++ b/exporter/prometheusexporter/go.mod @@ -24,6 +24,7 @@ require ( go.opentelemetry.io/collector/consumer v1.37.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/collector/exporter v0.131.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/collector/exporter/exportertest v0.131.1-0.20250801020258-8b73477b9810 + go.opentelemetry.io/collector/featuregate v1.37.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/collector/pdata v1.37.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/collector/receiver/receivertest v0.131.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/otel v1.37.0 @@ -206,7 +207,6 @@ require ( go.opentelemetry.io/collector/extension/extensionauth v1.37.1-0.20250801020258-8b73477b9810 // indirect go.opentelemetry.io/collector/extension/extensionmiddleware v0.131.1-0.20250801020258-8b73477b9810 // indirect go.opentelemetry.io/collector/extension/xextension v0.131.1-0.20250801020258-8b73477b9810 // indirect - go.opentelemetry.io/collector/featuregate v1.37.1-0.20250801020258-8b73477b9810 // indirect go.opentelemetry.io/collector/internal/telemetry v0.131.1-0.20250801020258-8b73477b9810 // indirect go.opentelemetry.io/collector/pdata/pprofile v0.131.1-0.20250801020258-8b73477b9810 // indirect go.opentelemetry.io/collector/pdata/xpdata v0.131.1-0.20250801020258-8b73477b9810 // indirect diff --git a/exporter/prometheusexporter/prometheus_test.go b/exporter/prometheusexporter/prometheus_test.go index e40fa6f754f68..f9f759c2ac122 100644 --- a/exporter/prometheusexporter/prometheus_test.go +++ b/exporter/prometheusexporter/prometheus_test.go @@ -453,7 +453,7 @@ func metricBuilder(delta int64, prefix, job, instance string) pmetric.Metrics { m1 := ms.AppendEmpty() m1.SetName(prefix + "this/one/there(where)") m1.SetDescription("Extra ones") - m1.SetUnit("1") + m1.SetUnit("By") d1 := m1.SetEmptySum() d1.SetIsMonotonic(true) d1.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) @@ -467,7 +467,7 @@ func metricBuilder(delta int64, prefix, job, instance string) pmetric.Metrics { m2 := ms.AppendEmpty() m2.SetName(prefix + "this/one/there(where)") m2.SetDescription("Extra ones") - m2.SetUnit("1") + m2.SetUnit("By") d2 := m2.SetEmptySum() d2.SetIsMonotonic(true) d2.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) @@ -480,3 +480,220 @@ func metricBuilder(delta int64, prefix, job, instance string) pmetric.Metrics { return md } + +func TestPrometheusExporter_TranslationStrategies(t *testing.T) { + tests := []struct { + name string + featureGateEnabled bool + config *Config + extraHeaders map[string]string + want string + }{ + { + name: "Legacy AddMetricSuffixes=true (no translation_strategy set)", + featureGateEnabled: false, + config: &Config{ + AddMetricSuffixes: true, + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP this_one_there_where_bytes_total Extra ones +# TYPE this_one_there_where_bytes_total counter +this_one_there_where_bytes_total{arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +this_one_there_where_bytes_total{arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + { + name: "Legacy AddMetricSuffixes=false (no translation_strategy set)", + featureGateEnabled: false, + config: &Config{ + AddMetricSuffixes: false, + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP this_one_there_where Extra ones +# TYPE this_one_there_where counter +this_one_there_where{arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +this_one_there_where{arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + { + name: "Legacy AddMetricSuffixes=true with feature gate enabled (no translation_strategy set)", + featureGateEnabled: true, + config: &Config{ + AddMetricSuffixes: true, // Should be ignored and default 'translation_strategy' is used (UnderscoreEscapingWithSuffixes). + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP this_one_there_where_bytes_total Extra ones +# TYPE this_one_there_where_bytes_total counter +this_one_there_where_bytes_total{arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +this_one_there_where_bytes_total{arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + { + name: "TranslationStrategy takes precedence over AddMetricSuffixes (feature gate disabled)", + featureGateEnabled: false, + config: &Config{ + AddMetricSuffixes: true, // This should be ignored + TranslationStrategy: underscoreEscapingWithoutSuffixes, + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP this_one_there_where Extra ones +# TYPE this_one_there_where counter +this_one_there_where{arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +this_one_there_where{arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + { + name: "UnderscoreEscapingWithSuffixes", + config: &Config{ + TranslationStrategy: underscoreEscapingWithSuffixes, + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP this_one_there_where_bytes_total Extra ones +# TYPE this_one_there_where_bytes_total counter +this_one_there_where_bytes_total{arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +this_one_there_where_bytes_total{arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + { + name: "UnderscoreEscapingWithoutSuffixes", + config: &Config{ + TranslationStrategy: underscoreEscapingWithoutSuffixes, + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP this_one_there_where Extra ones +# TYPE this_one_there_where counter +this_one_there_where{arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +this_one_there_where{arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + { + name: "NoUTF8EscapingWithSuffixes/escaping=allow-utf-8", + config: &Config{ + TranslationStrategy: noUTF8EscapingWithSuffixes, + }, + extraHeaders: map[string]string{ + "Accept": "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.6,application/openmetrics-text;version=0.0.1;q=0.5,text/plain;version=1.0.0;escaping=allow-utf-8;q=0.4,text/plain;version=0.0.4;q=0.3,*/*;q=0.2", + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP "this/one/there(where)_bytes_total" Extra ones +# TYPE "this/one/there(where)_bytes_total" counter +{"this/one/there(where)_bytes_total",arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +{"this/one/there(where)_bytes_total",arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + { + name: "NoUTF8EscapingWithSuffixes/escaping=underscores", + config: &Config{ + TranslationStrategy: noUTF8EscapingWithSuffixes, + }, + extraHeaders: map[string]string{ + "Accept": "application/openmetrics-text;version=1.0.0;escaping=underscores;q=0.5,application/openmetrics-text;version=0.0.1;q=0.4,text/plain;version=1.0.0;escaping=underscores;q=0.3,text/plain;version=0.0.4;q=0.2,/;q=0.1", + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP this_one_there_where__bytes_total Extra ones +# TYPE this_one_there_where__bytes_total counter +this_one_there_where__bytes_total{arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +this_one_there_where__bytes_total{arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + { + name: "NoTranslation/escaping=allow-utf-8", + featureGateEnabled: true, + config: &Config{ + TranslationStrategy: noTranslation, + }, + extraHeaders: map[string]string{ + "Accept": "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.6,application/openmetrics-text;version=0.0.1;q=0.5,text/plain;version=1.0.0;escaping=allow-utf-8;q=0.4,text/plain;version=0.0.4;q=0.3,*/*;q=0.2", + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP "this/one/there(where)" Extra ones +# TYPE "this/one/there(where)" counter +{"this/one/there(where)",arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +{"this/one/there(where)",arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + { + name: "NoTranslation/escaping=underscores", + config: &Config{ + TranslationStrategy: noTranslation, + }, + extraHeaders: map[string]string{ + "Accept": "application/openmetrics-text;version=1.0.0;escaping=underscores;q=0.5,application/openmetrics-text;version=0.0.1;q=0.4,text/plain;version=1.0.0;escaping=underscores;q=0.3,text/plain;version=0.0.4;q=0.2,/;q=0.1", + }, + want: `# HELP target_info Target metadata +# TYPE target_info gauge +target_info{instance="test-instance",job="test-service"} 1 +# HELP this_one_there_where_ Extra ones +# TYPE this_one_there_where_ counter +this_one_there_where_{arch="x86",instance="test-instance",job="test-service",os="linux",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 100 +this_one_there_where_{arch="x86",instance="test-instance",job="test-service",os="windows",otel_scope_name="",otel_scope_schema_url="",otel_scope_version=""} 99 +`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set feature gate state for this test + originalState := disableAddMetricSuffixesFeatureGate.IsEnabled() + testutil.SetFeatureGateForTest(t, disableAddMetricSuffixesFeatureGate, tt.featureGateEnabled) + defer testutil.SetFeatureGateForTest(t, disableAddMetricSuffixesFeatureGate, originalState) + + // Configure the exporter + addr := testutil.GetAvailableLocalAddress(t) + cfg := tt.config + cfg.ServerConfig = confighttp.ServerConfig{ + Endpoint: addr, + } + cfg.MetricExpiration = 120 * time.Minute + + factory := NewFactory() + set := exportertest.NewNopSettings(metadata.Type) + exp, err := factory.CreateMetrics(context.Background(), set, cfg) + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, exp.Shutdown(context.Background())) + }) + + assert.NotNil(t, exp) + require.NoError(t, exp.Start(context.Background(), componenttest.NewNopHost())) + + md := metricBuilder(0, "", "test-service", "test-instance") + assert.NoError(t, exp.ConsumeMetrics(context.Background(), md)) + + // Scrape metrics, with the Accept header set to the value specified in the test case + req, err := http.NewRequest(http.MethodGet, "http://"+addr+"/metrics", http.NoBody) + require.NoError(t, err) + for k, v := range tt.extraHeaders { + req.Header.Set(k, v) + } + res, err := http.DefaultClient.Do(req) + require.NoError(t, err, "Failed to perform a scrape") + assert.Equal(t, http.StatusOK, res.StatusCode, "Mismatched HTTP response status code") + + blob, _ := io.ReadAll(res.Body) + _ = res.Body.Close() + output := string(blob) + + assert.Equal(t, tt.want, output) + }) + } +} diff --git a/exporter/pulsarexporter/config.go b/exporter/pulsarexporter/config.go index 8b7be7e09d19b..7dcaa06ac539f 100644 --- a/exporter/pulsarexporter/config.go +++ b/exporter/pulsarexporter/config.go @@ -10,6 +10,7 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configoptional" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter/exporterhelper" ) @@ -39,10 +40,10 @@ type Config struct { } type Authentication struct { - TLS *TLS `mapstructure:"tls"` - Token *Token `mapstructure:"token"` - Athenz *Athenz `mapstructure:"athenz"` - OAuth2 *OAuth2 `mapstructure:"oauth2"` + TLS configoptional.Optional[TLS] `mapstructure:"tls"` + Token configoptional.Optional[Token] `mapstructure:"token"` + Athenz configoptional.Optional[Athenz] `mapstructure:"athenz"` + OAuth2 configoptional.Optional[OAuth2] `mapstructure:"oauth2"` } type TLS struct { @@ -95,28 +96,32 @@ func (*Config) Validate() error { func (cfg *Config) auth() pulsar.Authentication { authentication := cfg.Authentication - if authentication.TLS != nil { - return pulsar.NewAuthenticationTLS(authentication.TLS.CertFile, authentication.TLS.KeyFile) + if authentication.TLS.HasValue() { + tlsCfg := authentication.TLS.Get() + return pulsar.NewAuthenticationTLS(tlsCfg.CertFile, tlsCfg.KeyFile) } - if authentication.Token != nil { - return pulsar.NewAuthenticationToken(string(authentication.Token.Token)) + if authentication.Token.HasValue() { + tokenCfg := authentication.Token.Get() + return pulsar.NewAuthenticationToken(string(tokenCfg.Token)) } - if authentication.OAuth2 != nil { + if authentication.OAuth2.HasValue() { + oauth2Cfg := authentication.OAuth2.Get() return pulsar.NewAuthenticationOAuth2(map[string]string{ - "issuerUrl": authentication.OAuth2.IssuerURL, - "clientId": authentication.OAuth2.ClientID, - "audience": authentication.OAuth2.Audience, + "issuerUrl": oauth2Cfg.IssuerURL, + "clientId": oauth2Cfg.ClientID, + "audience": oauth2Cfg.Audience, }) } - if authentication.Athenz != nil { + if authentication.Athenz.HasValue() { + athenzCfg := authentication.Athenz.Get() return pulsar.NewAuthenticationAthenz(map[string]string{ - "providerDomain": authentication.Athenz.ProviderDomain, - "tenantDomain": authentication.Athenz.TenantDomain, - "tenantService": authentication.Athenz.TenantService, - "privateKey": string(authentication.Athenz.PrivateKey), - "keyId": authentication.Athenz.KeyID, - "principalHeader": authentication.Athenz.PrincipalHeader, - "ztsUrl": authentication.Athenz.ZtsURL, + "providerDomain": athenzCfg.ProviderDomain, + "tenantDomain": athenzCfg.TenantDomain, + "tenantService": athenzCfg.TenantService, + "privateKey": string(athenzCfg.PrivateKey), + "keyId": athenzCfg.KeyID, + "principalHeader": athenzCfg.PrincipalHeader, + "ztsUrl": athenzCfg.ZtsURL, }) } diff --git a/exporter/pulsarexporter/config_test.go b/exporter/pulsarexporter/config_test.go index b2b6047e3a902..a09fba7118551 100644 --- a/exporter/pulsarexporter/config_test.go +++ b/exporter/pulsarexporter/config_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configoptional" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/confmap/confmaptest" "go.opentelemetry.io/collector/confmap/xconfmap" @@ -55,7 +56,7 @@ func TestLoadConfig(t *testing.T) { Topic: "spans", Encoding: "otlp-spans", TLSTrustCertsFilePath: "ca.pem", - Authentication: Authentication{TLS: &TLS{CertFile: "cert.pem", KeyFile: "key.pem"}}, + Authentication: Authentication{TLS: configoptional.Some(TLS{CertFile: "cert.pem", KeyFile: "key.pem"})}, MaxConnectionsPerBroker: 1, ConnectionTimeout: 5 * time.Second, OperationTimeout: 30 * time.Second, diff --git a/exporter/pulsarexporter/go.mod b/exporter/pulsarexporter/go.mod index 3cb4b6952009b..1c2a9e2e49eaf 100644 --- a/exporter/pulsarexporter/go.mod +++ b/exporter/pulsarexporter/go.mod @@ -13,6 +13,7 @@ require ( go.opentelemetry.io/collector/component v1.37.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/collector/component/componenttest v0.131.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/collector/config/configopaque v1.37.1-0.20250801020258-8b73477b9810 + go.opentelemetry.io/collector/config/configoptional v0.131.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/collector/config/configretry v1.37.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/collector/confmap v1.37.1-0.20250801020258-8b73477b9810 go.opentelemetry.io/collector/confmap/xconfmap v0.131.1-0.20250801020258-8b73477b9810 @@ -82,7 +83,6 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/collector/client v1.37.1-0.20250801020258-8b73477b9810 // indirect - go.opentelemetry.io/collector/config/configoptional v0.131.1-0.20250801020258-8b73477b9810 // indirect go.opentelemetry.io/collector/consumer/consumertest v0.131.1-0.20250801020258-8b73477b9810 // indirect go.opentelemetry.io/collector/consumer/xconsumer v0.131.1-0.20250801020258-8b73477b9810 // indirect go.opentelemetry.io/collector/exporter/xexporter v0.131.1-0.20250801020258-8b73477b9810 // indirect diff --git a/extension/datadogextension/go.mod b/extension/datadogextension/go.mod index 3352dee55942f..4554dcca55398 100644 --- a/extension/datadogextension/go.mod +++ b/extension/datadogextension/go.mod @@ -81,7 +81,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/sort v0.67.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.67.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 // indirect github.com/DataDog/datadog-agent/pkg/version v0.67.0 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.42.0 // indirect github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee // indirect @@ -277,7 +277,7 @@ require ( golang.org/x/sys v0.34.0 // indirect golang.org/x/term v0.33.0 // indirect golang.org/x/text v0.27.0 // indirect - golang.org/x/time v0.11.0 // indirect + golang.org/x/time v0.12.0 // indirect gonum.org/v1/gonum v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect diff --git a/extension/datadogextension/go.sum b/extension/datadogextension/go.sum index 8ef0fcb4b628b..c115f55ceb8f1 100644 --- a/extension/datadogextension/go.sum +++ b/extension/datadogextension/go.sum @@ -130,8 +130,8 @@ github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 h1:pM5p51BfBYgkp github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0/go.mod h1:cQWaDB4GxK3i+0jbXZlePXzJ3f1oz2bnEPVfF9rK1Dg= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0 h1:mLXT+D29nRHZdUHQvLp+mskZ9kQZJqkkk05EsEjw7WY= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0/go.mod h1:yi2rztz2PWPdqIykyyX4nHkJiQLvEn0dTPzj9V7g7o8= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 h1:as8nOTMEeifcLCeHN+oqfdRrtUtVZIvwIN7kD2fQDIM= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0/go.mod h1:2l1nTFRygeIQkYr8E1tHuO/ZMjxPt0VgJ1wW14F2Fb4= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 h1:8euFCQn/SNpuN3L5uHx0bHj3N6Yf4QBukUSdylwArSA= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608/go.mod h1:mKV8FnTNGRSJF/TEPDOYWCCnarpPsxhfTZBVkFh/zRY= github.com/DataDog/datadog-agent/pkg/version v0.67.0 h1:TB8H8r+laB1Qdttvvc6XJVyLGxp8E6j2f2Mh5IPbYmQ= github.com/DataDog/datadog-agent/pkg/version v0.67.0/go.mod h1:kvAw/WbI7qLAsDI2wHabZfM7Cv2zraD3JA3323GEB+8= github.com/DataDog/datadog-api-client-go/v2 v2.42.0 h1:0L03LqChbOf7IaaiBUZpXi1Ss6PseGGhQEQrNqD9nU8= @@ -827,8 +827,8 @@ golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/datadog/go.mod b/internal/datadog/go.mod index a49c92e119394..1024f1e1227c7 100644 --- a/internal/datadog/go.mod +++ b/internal/datadog/go.mod @@ -63,7 +63,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.67.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 // indirect github.com/DataDog/datadog-agent/pkg/version v0.67.0 // indirect github.com/DataDog/sketches-go v1.4.7 // indirect github.com/DataDog/viper v1.14.0 // indirect @@ -187,7 +187,7 @@ require ( golang.org/x/sys v0.34.0 // indirect golang.org/x/term v0.33.0 // indirect golang.org/x/text v0.27.0 // indirect - golang.org/x/time v0.11.0 // indirect + golang.org/x/time v0.12.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect google.golang.org/grpc v1.74.2 // indirect google.golang.org/protobuf v1.36.6 // indirect diff --git a/internal/datadog/go.sum b/internal/datadog/go.sum index f5667c31dcadc..256ee60fe2228 100644 --- a/internal/datadog/go.sum +++ b/internal/datadog/go.sum @@ -72,8 +72,8 @@ github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 h1:pM5p51BfBYgkp github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0/go.mod h1:cQWaDB4GxK3i+0jbXZlePXzJ3f1oz2bnEPVfF9rK1Dg= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0 h1:mLXT+D29nRHZdUHQvLp+mskZ9kQZJqkkk05EsEjw7WY= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0/go.mod h1:yi2rztz2PWPdqIykyyX4nHkJiQLvEn0dTPzj9V7g7o8= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 h1:as8nOTMEeifcLCeHN+oqfdRrtUtVZIvwIN7kD2fQDIM= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0/go.mod h1:2l1nTFRygeIQkYr8E1tHuO/ZMjxPt0VgJ1wW14F2Fb4= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 h1:8euFCQn/SNpuN3L5uHx0bHj3N6Yf4QBukUSdylwArSA= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608/go.mod h1:mKV8FnTNGRSJF/TEPDOYWCCnarpPsxhfTZBVkFh/zRY= github.com/DataDog/datadog-agent/pkg/version v0.67.0 h1:TB8H8r+laB1Qdttvvc6XJVyLGxp8E6j2f2Mh5IPbYmQ= github.com/DataDog/datadog-agent/pkg/version v0.67.0/go.mod h1:kvAw/WbI7qLAsDI2wHabZfM7Cv2zraD3JA3323GEB+8= github.com/DataDog/datadog-api-client-go/v2 v2.42.0 h1:0L03LqChbOf7IaaiBUZpXi1Ss6PseGGhQEQrNqD9nU8= @@ -656,8 +656,8 @@ golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/datadog/go.mod b/pkg/datadog/go.mod index 0e16757b995ac..46052ed94da93 100644 --- a/pkg/datadog/go.mod +++ b/pkg/datadog/go.mod @@ -82,7 +82,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/sort v0.67.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.67.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 // indirect github.com/DataDog/datadog-agent/pkg/version v0.67.0 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.42.0 // indirect github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee // indirect @@ -236,7 +236,7 @@ require ( golang.org/x/sys v0.34.0 // indirect golang.org/x/term v0.33.0 // indirect golang.org/x/text v0.27.0 // indirect - golang.org/x/time v0.11.0 // indirect + golang.org/x/time v0.12.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect google.golang.org/grpc v1.74.2 // indirect google.golang.org/protobuf v1.36.6 // indirect diff --git a/pkg/datadog/go.sum b/pkg/datadog/go.sum index 30686087ff7f3..55cc6fede4a9e 100644 --- a/pkg/datadog/go.sum +++ b/pkg/datadog/go.sum @@ -130,8 +130,8 @@ github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 h1:pM5p51BfBYgkp github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0/go.mod h1:cQWaDB4GxK3i+0jbXZlePXzJ3f1oz2bnEPVfF9rK1Dg= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0 h1:mLXT+D29nRHZdUHQvLp+mskZ9kQZJqkkk05EsEjw7WY= github.com/DataDog/datadog-agent/pkg/util/testutil v0.67.0/go.mod h1:yi2rztz2PWPdqIykyyX4nHkJiQLvEn0dTPzj9V7g7o8= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 h1:as8nOTMEeifcLCeHN+oqfdRrtUtVZIvwIN7kD2fQDIM= -github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0/go.mod h1:2l1nTFRygeIQkYr8E1tHuO/ZMjxPt0VgJ1wW14F2Fb4= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 h1:8euFCQn/SNpuN3L5uHx0bHj3N6Yf4QBukUSdylwArSA= +github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608/go.mod h1:mKV8FnTNGRSJF/TEPDOYWCCnarpPsxhfTZBVkFh/zRY= github.com/DataDog/datadog-agent/pkg/version v0.67.0 h1:TB8H8r+laB1Qdttvvc6XJVyLGxp8E6j2f2Mh5IPbYmQ= github.com/DataDog/datadog-agent/pkg/version v0.67.0/go.mod h1:kvAw/WbI7qLAsDI2wHabZfM7Cv2zraD3JA3323GEB+8= github.com/DataDog/datadog-api-client-go/v2 v2.42.0 h1:0L03LqChbOf7IaaiBUZpXi1Ss6PseGGhQEQrNqD9nU8= @@ -755,8 +755,8 @@ golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/receiver/datadogreceiver/go.mod b/receiver/datadogreceiver/go.mod index efee21176747f..ae926903936d8 100644 --- a/receiver/datadogreceiver/go.mod +++ b/receiver/datadogreceiver/go.mod @@ -138,7 +138,7 @@ require ( golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect - golang.org/x/time v0.11.0 // indirect + golang.org/x/time v0.12.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect google.golang.org/grpc v1.74.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/receiver/datadogreceiver/go.sum b/receiver/datadogreceiver/go.sum index b61150ab63642..d75f10357e312 100644 --- a/receiver/datadogreceiver/go.sum +++ b/receiver/datadogreceiver/go.sum @@ -368,8 +368,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/receiver/tlscheckreceiver/scraper.go b/receiver/tlscheckreceiver/scraper.go index 805e8da072326..f2ddfda2c86ef 100644 --- a/receiver/tlscheckreceiver/scraper.go +++ b/receiver/tlscheckreceiver/scraper.go @@ -124,7 +124,7 @@ func (s *scraper) scrapeEndpoint(endpoint string, metrics *pmetric.Metrics, wg * return } - s.settings.Logger.Info("Peer Certificates", zap.Int("certificates_count", len(state.PeerCertificates))) + s.settings.Logger.Debug("Peer Certificates", zap.Int("certificates_count", len(state.PeerCertificates))) if len(state.PeerCertificates) == 0 { err := fmt.Errorf("no TLS certificates found for endpoint: %s. Verify the endpoint serves TLS certificates", endpoint) s.settings.Logger.Error(err.Error(), zap.String("endpoint", endpoint)) diff --git a/testbed/go.mod b/testbed/go.mod index 7bdb31313ea59..788f7cd66e81e 100644 --- a/testbed/go.mod +++ b/testbed/go.mod @@ -142,7 +142,7 @@ require ( github.com/dennwc/varint v1.0.0 // indirect github.com/digitalocean/godo v1.152.0 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v28.2.2+incompatible // indirect + github.com/docker/docker v28.3.3+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/testbed/go.sum b/testbed/go.sum index 5072c61f06664..1457e9169e62d 100644 --- a/testbed/go.sum +++ b/testbed/go.sum @@ -192,8 +192,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=