From 933d8d78283ea7fe3cf39ff47f9a59dd1e180e99 Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Mon, 27 Jul 2026 10:54:46 -0700 Subject: [PATCH] fix(byoo): cap chunked log exporter batches Configure exporterhelper byte batching at 900,000 bytes when log chunking is active, leaving space for request envelope overhead. Preserve dry-run behavior and bump the BYOO collector release to 0.157.1. Signed-off-by: Kristina Pathak --- .../byoo-otel-collector/VERSION | 2 +- .../internal/otelconfig/render.go | 18 ++++++++++++++++++ .../internal/otelconfig/render_test.go | 18 ++++++++++++++---- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/compute-plane-services/byoo-otel-collector/VERSION b/src/compute-plane-services/byoo-otel-collector/VERSION index a95410b17..4f58d4880 100644 --- a/src/compute-plane-services/byoo-otel-collector/VERSION +++ b/src/compute-plane-services/byoo-otel-collector/VERSION @@ -1 +1 @@ -0.157.0 +0.157.1 diff --git a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go index ef50aed04..cffc0e858 100644 --- a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go +++ b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go @@ -68,6 +68,9 @@ type OpenTelemetryConfig struct { const ( defaultLogChunkMaxPayloadBytes = 262144 minConfiguredLogChunkMaxPayloadBytes = 4 + defaultLogExporterBatchFlushTimeout = "200ms" + // Leave 100 KB below the 1 MB receiver limit for the export envelope. + defaultLogExporterBatchSizeBytes = int64(900_000) ) const ( @@ -267,6 +270,18 @@ func logExporterSendingQueue() map[string]interface{} { } } +func enableChunkedLogExporterBatching(otelConfig *OpenTelemetryConfig, exporterID string) { + exporter := otelConfig.Exporters[exporterID] + queue := mapFromInterface(exporter["sending_queue"]) + queue["batch"] = map[string]interface{}{ + "flush_timeout": defaultLogExporterBatchFlushTimeout, + "sizer": "bytes", + "min_size": defaultLogExporterBatchSizeBytes, + "max_size": defaultLogExporterBatchSizeBytes, + } + exporter["sending_queue"] = queue +} + func exporterLogs(config TelemetryConfig, otelConfig *OpenTelemetryConfig) (exporterId string, err error) { var exporterType, exporterName string var exporterCredential interface{} @@ -857,6 +872,9 @@ func generateExportersAndService(config TelemetryConfig, otelConfig *OpenTelemet "dry_run": logChunking.DryRun, } logPipeline.Processors = append(logPipeline.Processors, "logchunk/byoo") + if !logChunking.DryRun { + enableChunkedLogExporterBatching(otelConfig, exporterId) + } } logPipeline.Processors = append(logPipeline.Processors, "batch") otelConfig.Service.Pipelines["logs"] = logPipeline diff --git a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go index 557e27c5f..2768b0f07 100644 --- a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go +++ b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go @@ -482,8 +482,8 @@ func TestGenerateExportersAndServiceAddsLogChunkDefaultsWhenEnabled(t *testing.T Logs: &Telemetry{ Name: "example-logs", Protocol: ProtocolHTTP, - Provider: ProviderSplunk, - Endpoint: "https://splunk.example.invalid", + Provider: ProviderKratosLogs, + Endpoint: "https://kratos.example.invalid", }, }, } @@ -502,8 +502,18 @@ func TestGenerateExportersAndServiceAddsLogChunkDefaultsWhenEnabled(t *testing.T "max_payload_bytes": defaultLogChunkMaxPayloadBytes, "dry_run": false, }, otelConfig.Processors["logchunk/byoo"]) - exporter := otelConfig.Exporters["splunk_hec/SPLUNK-example-logs-logs"] - assert.NotContains(t, exporter["sending_queue"].(map[string]interface{}), "batch") + exporter := otelConfig.Exporters["otlp_http/KRATOS-example-logs-logs"] + assert.Equal(t, map[string]interface{}{ + "enabled": true, + "num_consumers": 10, + "queue_size": 1000, + "batch": map[string]interface{}{ + "flush_timeout": defaultLogExporterBatchFlushTimeout, + "sizer": "bytes", + "min_size": defaultLogExporterBatchSizeBytes, + "max_size": defaultLogExporterBatchSizeBytes, + }, + }, exporter["sending_queue"]) } func TestGenerateExportersAndServiceUsesExporterHelperQueueBatchConfig(t *testing.T) {