[chore] merge main into branch

.chloggen/esexporter-queuebatch.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: deprecation
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: elasticsearchexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Allow `sending_queue` to queue and batch based on the new API and deprecate `batcher` config
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [41338]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.chloggen/mx-psi_configoptional-exporter-pulsar.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pulsarexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Use `configoptional.Optional` for authentication fields
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [41723]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

.chloggen/prometheusexporter-translation-strategies.yaml

@@ -0,0 +1,34 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: exporter/prometheus
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add `translation_strategy` configuration option to control how OTLP metric names are translated to Prometheus format.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [35459]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  The new `translation_strategy` option provides four different translation modes:
+  - `UnderscoreEscapingWithSuffixes`: Escapes special characters to underscores and appends type/unit suffixes
+  - `UnderscoreEscapingWithoutSuffixes`: Escapes special characters but omits suffixes
+  - `NoUTF8EscapingWithSuffixes`: Preserves UTF-8 characters while adding suffixes
+  - `NoTranslation`: Passes metric names through unaltered
+  When `translation_strategy` is set, it always takes precedence over the deprecated `add_metric_suffixes` option.
+  The `exporter.prometheusexporter.DisableAddMetricSuffixes` feature gate can be used to completely ignore the deprecated `add_metric_suffixes` setting.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

connector/datadogconnector/go.mod

@@ -122,7 +122,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/util/statstracker v0.66.1 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/system v0.67.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 // indirect
 	github.com/DataDog/datadog-agent/pkg/version v0.68.0 // indirect
 	github.com/DataDog/datadog-api-client-go/v2 v2.43.0 // indirect
 	github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect

exporter/clickhouseexporter/go.mod

@@ -37,7 +37,7 @@ require (
 	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
-	github.com/docker/docker v28.3.2+incompatible // indirect
+	github.com/docker/docker v28.3.3+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/ebitengine/purego v0.8.4 // indirect

exporter/datadogexporter/go.mod

@@ -149,7 +149,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/util/statstracker v0.66.1 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/system v0.67.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 // indirect
 	github.com/DataDog/datadog-agent/pkg/version v0.68.0 // indirect
 	github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect
 	github.com/DataDog/go-sqllexer v0.1.6 // indirect

exporter/datadogexporter/integrationtest/go.mod

@@ -123,7 +123,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/util/statstracker v0.66.1 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/system v0.67.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/system/socket v0.67.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/util/winutil v0.67.0 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/winutil v0.70.0-devel.0.20250801132403-9300d3e37608 // indirect
 	github.com/DataDog/datadog-agent/pkg/version v0.68.0 // indirect
 	github.com/DataDog/datadog-api-client-go/v2 v2.43.0 // indirect
 	github.com/DataDog/datadog-go/v5 v5.6.0 // indirect

exporter/elasticsearchexporter/README.md

@@ -82,14 +82,45 @@ As a consequence of supporting [confighttp], the Elasticsearch exporter also sup
 The Elasticsearch exporter sets `timeout` (HTTP request timeout) to 90s by default.
 All other defaults are as defined by [confighttp].
 
-### Queuing
+### Queuing and batching
 
-The Elasticsearch exporter supports the common [`sending_queue` settings][exporterhelper]. However, the sending queue is currently disabled by default.
+The exporter is transitioning from its own internal batching to OpenTelemetry's standard
+queueing and batching. The below sections describe the current default and the latest
+configuration option for queueing and batching available via the `sending_queue` configuration.
 
-### Batching
+#### Internal batching by Elasticsearch exporter
+
+By default, the exporter will perform its own buffering and batching, as configured through the
+`flush` config. In this case both `sending_queue` and `batcher` will be unused. The exporter
+will perform its own buffering and batching and will issue async requests to Elasticsearch in
+all cases other than if any of the following conditions are met:
+
+- `sending_queue::batch` is defined (irrespective of `sending_queue` being enabled or not)
+- `batcher::enabled` is defined (set to `true` or `false`)
+
+In a future release when the `sending_queue` config is stable, and has feature parity
+with the exporter's existing `flush` config, it will be enabled by default.
+
+Using the `sending_queue` functionality provides several benefits over the default behavior:
+ - With a persistent queue, or no queue at all, `sending_queue` enables at least once delivery.
+   On the other hand, with the default behavior, the exporter will accept data and process it
+   asynchronously, which interacts poorly with queueing.
+ - By ensuring the exporter makes requests to Elasticsearch synchronously (batching disabled),
+   client metadata can be passed through to Elasticsearch requests,
+   e.g. by using the [`headers_setter` extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/extension/headerssetterextension/README.md).
+
+#### Queueing and batching using sending queue
+
+The Elasticsearch exporter supports the common [`sending_queue` settings][exporterhelper] which
+supports both queueing and batching. However, the sending queue is currently disabled by
+default. Sending queue can be enabled by setting `sending_queue::enabled` to `true`. The batching support in sending queue is also disabled by default. Batching can be enabled by defining `sending_queue::batch`.
+
+The [`exporterhelper` documentation][exporterhelper] provides more details on the `sending_queue` settings.
+
+#### Deprecated batcher config
 
 > [!WARNING]
-> The `batcher` config is experimental and may change without notice.
+> The `batcher` config is now deprecated and will be removed in an upcoming version. Check the [queueing and batching](#queueing-and-batching) section for using the `sending_queue` setting that supersedes `batcher`. In the interim, `batcher` configurations are still valid, however, they will be ignored if `sending_queue::batch` is defined even if `sending_queue` is not enabled.
 
 The Elasticsearch exporter supports the [common `batcher` settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/internal/queue_sender.go).
 
@@ -100,21 +131,6 @@ The Elasticsearch exporter supports the [common `batcher` settings](https://gith
   - `max_size` (default=0): Maximum batch size to be exported to Elasticsearch, measured in units according to `batcher::sizer`. To limit bulk request size, configure `flush::bytes` instead. :warning: It is recommended to keep `max_size` as 0 as a non-zero value may lead to broken metrics grouping and indexing rejections.
   - `flush_timeout` (default=30s): Maximum time of the oldest item spent inside the batcher buffer, aka "max age of batcher buffer". A batcher flush will happen regardless of the size of content in batcher buffer.
 
-By default, the exporter will perform its own buffering and batching, as configured through the
-`flush` config, and `batcher` will be unused. By setting `batcher::enabled` to either `true` or
-`false`, the exporter will not perform any of its own buffering or batching, and the `flush::interval` config
-will be ignored.
-In a future release when the `batcher` config is stable, and has feature parity
-with the exporter's existing `flush` config, it will be enabled by default.
-
-Using the common `batcher` functionality provides several benefits over the default behavior:
- - Combined with a persistent queue, or no queue at all, `batcher` enables at least once delivery.
-   With the default behavior, the exporter will accept data and process it asynchronously,
-   which interacts poorly with queuing.
- - By ensuring the exporter makes requests to Elasticsearch synchronously,
-   client metadata can be passed through to Elasticsearch requests,
-   e.g. by using the [`headers_setter` extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/extension/headerssetterextension/README.md).
-
 ### Elasticsearch document routing
 
 Documents are statically or dynamically routed to the target index / data stream in the following order. The first routing mode that applies will be used.
@@ -359,11 +375,18 @@ The Elasticsearch Exporter's own telemetry settings for testing and debugging pu
 
 ### Metadata keys
 
-Metadata keys are a list of client metadata keys that the exporter currently uses to enrich internal telemetry.
+Metadata keys are a list of client metadata keys that the exporter uses to partition batches
+when `sending_queue` is enabled with batching support and enrich internal telemetry.
 
 ⚠️ This is experimental and may change at any time.
 
-- `metadata_keys` (optional): List of metadata keys that will be added to the exporter's telemetry if defined. The config only applies when batcher is used (set to `true` or `false`). The metadata keys are converted to lower case as key lookups for client metadata is case insensitive. This means that the metric produced by internal telemetry will also have the attribute in lower case.
+- `metadata_keys` (optional): List of metadata keys that will be used to partition the data
+into batches if [sending_queue][exporterhelper] is enabled with batching support OR
+`batcher::enabled` is set. The keys will also be used to enrich the exporter's internal
+telemetry if defined. The keys are extracted from the client metadata available via the context
+and added to the internal telemetry as attributes.
+
+NOTE: The metadata keys are converted to lower case as key lookups for client metadata is case insensitive. This means that the metric produced by internal telemetry will also have the attribute in lower case.
 
 ## Exporting metrics
 

exporter/elasticsearchexporter/bulkindexer.go

@@ -71,7 +71,7 @@ func newBulkIndexer(
 	tb *metadata.TelemetryBuilder,
 	logger *zap.Logger,
 ) (bulkIndexer, error) {
-	if config.Batcher.enabledSet {
+	if config.Batcher.enabledSet || (config.QueueBatchConfig.Enabled && config.QueueBatchConfig.Batch.HasValue()) {
 		return newSyncBulkIndexer(client, config, requireDataStream, tb, logger), nil
 	}
 	return newAsyncBulkIndexer(client, config, requireDataStream, tb, logger)