[confgenerator ] Set block_on_overflow and exporter batching in Otel Cloud Logging exporter.

confgenerator/confgenerator.go

@@ -64,6 +64,22 @@ func googleCloudLoggingExporter() otel.Component {
 			// Set to mirror the 60s max limit of default retry window in Google Cloud Logging apiv2 go client :
 			// https://github.com/googleapis/google-cloud-go/blob/logging/v1.4.2/logging/apiv2/logging_client.go#L78-L90
 			"timeout": "60s",
+			"sending_queue": map[string]interface{}{
+				"enabled": true,
+				// Set queue_size to "(num_consumers + 2)*1000" to always have a new batch ready.
+				"queue_size":    12000,
+				"num_consumers": 10,
+				"sizer":         "items",
+				// Blocks the "sending_queue" on overflow to reduce log loss.
+				"block_on_overflow": true,
+				// Set batch in "sending_queue" is recommended instead of using the batch processor.
+				"batch": map[string]interface{}{
+					"flush_timeout": "200ms",
+					"min_size":      1000,
+					"max_size":      1000,
+					"sizer":         "items",
+				},
+			},
 		},
 	}
 }
@@ -228,12 +244,6 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDi
 			},
 			otel.Logging: {
 				Exporter: googleCloudLoggingExporter(),
-				ProcessorsByType: map[string][]otel.Component{
-					// Batching logs improves log export performance.
-					"logs": {
-						otel.BatchProcessor(1000, 1000, "200ms"),
-					},
-				},
 			},
 		},
 	}.Generate(ctx)

confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml

@@ -8,6 +8,17 @@ exporters:
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
   googlecloud/logging:
+    sending_queue:
+      batch:
+        flush_timeout: 200ms
+        max_size: 1000
+        min_size: 1000
+        sizer: items
+      block_on_overflow: true
+      enabled: true
+      num_consumers: 10
+      queue_size: 12000
+      sizer: items
     timeout: 60s
   googlecloud/otel:
     metric:
@@ -25,10 +36,6 @@ processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
     - system.cpu.utilization
-  batch/googlecloud/logging_logs_0:
-    send_batch_max_size: 1000
-    send_batch_size: 1000
-    timeout: 200ms
   cumulativetodelta/loggingmetrics_4:
     include:
       match_type: strict
@@ -742,7 +749,6 @@ service:
       processors:
       - transform/otlp_0
       - resourcedetection/_global_1
-      - batch/googlecloud/logging_logs_0
       receivers:
       - otlp/otlp
     metrics/default__pipeline_hostmetrics:

confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml

@@ -8,6 +8,17 @@ exporters:
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
   googlecloud/logging:
+    sending_queue:
+      batch:
+        flush_timeout: 200ms
+        max_size: 1000
+        min_size: 1000
+        sizer: items
+      block_on_overflow: true
+      enabled: true
+      num_consumers: 10
+      queue_size: 12000
+      sizer: items
     timeout: 60s
   googlecloud/otel:
     metric:
@@ -25,10 +36,6 @@ processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
     - system.cpu.utilization
-  batch/googlecloud/logging_logs_0:
-    send_batch_max_size: 1000
-    send_batch_size: 1000
-    timeout: 200ms
   cumulativetodelta/loggingmetrics_4:
     include:
       match_type: strict
@@ -711,7 +718,6 @@ service:
       processors:
       - transform/otlp_0
       - resourcedetection/_global_1
-      - batch/googlecloud/logging_logs_0
       receivers:
       - otlp/otlp
     metrics/default__pipeline_hostmetrics:

confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml

@@ -8,6 +8,17 @@ exporters:
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
   googlecloud/logging:
+    sending_queue:
+      batch:
+        flush_timeout: 200ms
+        max_size: 1000
+        min_size: 1000
+        sizer: items
+      block_on_overflow: true
+      enabled: true
+      num_consumers: 10
+      queue_size: 12000
+      sizer: items
     timeout: 60s
   googlecloud/otel:
     metric:
@@ -25,10 +36,6 @@ processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
     - system.cpu.utilization
-  batch/googlecloud/logging_logs_0:
-    send_batch_max_size: 1000
-    send_batch_size: 1000
-    timeout: 200ms
   casttosum/iis_1:
     metrics:
     - agent.googleapis.com/iis/network/transferred_bytes_count
@@ -824,7 +831,6 @@ service:
       processors:
       - transform/otlp_0
       - resourcedetection/_global_1
-      - batch/googlecloud/logging_logs_0
       receivers:
       - otlp/otlp
     metrics/default__pipeline_hostmetrics:

confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml

@@ -8,6 +8,17 @@ exporters:
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
   googlecloud/logging:
+    sending_queue:
+      batch:
+        flush_timeout: 200ms
+        max_size: 1000
+        min_size: 1000
+        sizer: items
+      block_on_overflow: true
+      enabled: true
+      num_consumers: 10
+      queue_size: 12000
+      sizer: items
     timeout: 60s
   googlecloud/otel:
     metric:
@@ -25,10 +36,6 @@ processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
     - system.cpu.utilization
-  batch/googlecloud/logging_logs_0:
-    send_batch_max_size: 1000
-    send_batch_size: 1000
-    timeout: 200ms
   casttosum/iis_1:
     metrics:
     - agent.googleapis.com/iis/network/transferred_bytes_count
@@ -824,7 +831,6 @@ service:
       processors:
       - transform/otlp_0
       - resourcedetection/_global_1
-      - batch/googlecloud/logging_logs_0
       receivers:
       - otlp/otlp
     metrics/default__pipeline_hostmetrics:

confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml

@@ -8,6 +8,17 @@ exporters:
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
   googlecloud/logging:
+    sending_queue:
+      batch:
+        flush_timeout: 200ms
+        max_size: 1000
+        min_size: 1000
+        sizer: items
+      block_on_overflow: true
+      enabled: true
+      num_consumers: 10
+      queue_size: 12000
+      sizer: items
     timeout: 60s
 extensions:
   file_storage:
@@ -17,10 +28,6 @@ processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
     - system.cpu.utilization
-  batch/googlecloud/logging_logs_0:
-    send_batch_max_size: 1000
-    send_batch_size: 1000
-    timeout: 200ms
   cumulativetodelta/loggingmetrics_4:
     include:
       match_type: strict
@@ -885,7 +892,6 @@ service:
       processors:
       - transform/syslog_0
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - filelog/syslog
     logs/logs_p1_sample__logs:
@@ -917,7 +923,6 @@ service:
       - filter/logs_p1_sample__logs_21
       - filter/logs_p1_sample__logs_22
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - filelog/sample__logs
     metrics/default__pipeline_hostmetrics:

confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml

@@ -8,6 +8,17 @@ exporters:
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
   googlecloud/logging:
+    sending_queue:
+      batch:
+        flush_timeout: 200ms
+        max_size: 1000
+        min_size: 1000
+        sizer: items
+      block_on_overflow: true
+      enabled: true
+      num_consumers: 10
+      queue_size: 12000
+      sizer: items
     timeout: 60s
 extensions:
   file_storage:
@@ -17,10 +28,6 @@ processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
     - system.cpu.utilization
-  batch/googlecloud/logging_logs_0:
-    send_batch_max_size: 1000
-    send_batch_size: 1000
-    timeout: 200ms
   cumulativetodelta/loggingmetrics_4:
     include:
       match_type: strict
@@ -854,7 +861,6 @@ service:
       processors:
       - transform/syslog_0
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - filelog/syslog
     logs/logs_p1_sample__logs:
@@ -886,7 +892,6 @@ service:
       - filter/logs_p1_sample__logs_21
       - filter/logs_p1_sample__logs_22
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - filelog/sample__logs
     metrics/default__pipeline_hostmetrics:

confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml

@@ -8,6 +8,17 @@ exporters:
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
   googlecloud/logging:
+    sending_queue:
+      batch:
+        flush_timeout: 200ms
+        max_size: 1000
+        min_size: 1000
+        sizer: items
+      block_on_overflow: true
+      enabled: true
+      num_consumers: 10
+      queue_size: 12000
+      sizer: items
     timeout: 60s
 extensions:
   file_storage:
@@ -17,10 +28,6 @@ processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
     - system.cpu.utilization
-  batch/googlecloud/logging_logs_0:
-    send_batch_max_size: 1000
-    send_batch_size: 1000
-    timeout: 200ms
   casttosum/iis_1:
     metrics:
     - agent.googleapis.com/iis/network/transferred_bytes_count
@@ -1308,7 +1315,6 @@ service:
       - transform/windows__event__log_1
       - transform/windows__event__log_2
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - windowseventlog/windows__event__log
     logs/logs_default__pipeline_windows__event__log_1:
@@ -1319,7 +1325,6 @@ service:
       - transform/windows__event__log_1_1
       - transform/windows__event__log_1_2
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - windowseventlog/windows__event__log_1
     logs/logs_default__pipeline_windows__event__log_2:
@@ -1330,7 +1335,6 @@ service:
       - transform/windows__event__log_2_1
       - transform/windows__event__log_2_2
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - windowseventlog/windows__event__log_2
     logs/logs_p1_sample__logs:
@@ -1362,7 +1366,6 @@ service:
       - filter/logs_p1_sample__logs_21
       - filter/logs_p1_sample__logs_22
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - filelog/sample__logs
     metrics/default__pipeline_hostmetrics:

confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml

@@ -8,6 +8,17 @@ exporters:
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
   googlecloud/logging:
+    sending_queue:
+      batch:
+        flush_timeout: 200ms
+        max_size: 1000
+        min_size: 1000
+        sizer: items
+      block_on_overflow: true
+      enabled: true
+      num_consumers: 10
+      queue_size: 12000
+      sizer: items
     timeout: 60s
 extensions:
   file_storage:
@@ -17,10 +28,6 @@ processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
     - system.cpu.utilization
-  batch/googlecloud/logging_logs_0:
-    send_batch_max_size: 1000
-    send_batch_size: 1000
-    timeout: 200ms
   casttosum/iis_1:
     metrics:
     - agent.googleapis.com/iis/network/transferred_bytes_count
@@ -1308,7 +1315,6 @@ service:
       - transform/windows__event__log_1
       - transform/windows__event__log_2
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - windowseventlog/windows__event__log
     logs/logs_default__pipeline_windows__event__log_1:
@@ -1319,7 +1325,6 @@ service:
       - transform/windows__event__log_1_1
       - transform/windows__event__log_1_2
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - windowseventlog/windows__event__log_1
     logs/logs_default__pipeline_windows__event__log_2:
@@ -1330,7 +1335,6 @@ service:
       - transform/windows__event__log_2_1
       - transform/windows__event__log_2_2
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - windowseventlog/windows__event__log_2
     logs/logs_p1_sample__logs:
@@ -1362,7 +1366,6 @@ service:
       - filter/logs_p1_sample__logs_21
       - filter/logs_p1_sample__logs_22
       - resourcedetection/_global_0
-      - batch/googlecloud/logging_logs_0
       receivers:
       - filelog/sample__logs
     metrics/default__pipeline_hostmetrics: