Skip to content
3 changes: 3 additions & 0 deletions src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
* Updated OpenTelemetry core component version(s) to `1.16.0`.
([#4487](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/4487))

* Log error when telemetry data exceeds the serialization buffer capacity.
([#4027](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/4027))

## 1.15.2

Released 2026-Apr-21
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,16 @@ public ExportResult Export(in Batch<LogRecord> batch)
try
{
var eventBuilder = this.SerializeLogRecord(logRecord);
eventBuilder.Write(this.logsTracepoint);
var errorCode = eventBuilder.Write(this.logsTracepoint);
if (errorCode != 0)
{
if (!GenevaBufferOverflowExceptionHelper.TryReportLogBufferOverflow(errorCode))
{
ExporterEventSource.Log.FailedToSendLogData($"EventHeader log export failed with errno {errorCode}.");
}

result = ExportResult.Failure;
}
}
catch (Exception ex)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,20 @@ internal sealed class ExporterEventSource : EventSource
private const int EVENT_ID_TRANSPORT_EXCEPTION = 8; // Transport exception
private const int EVENT_ID_TRANSPORT_INFO = 9; // Transport info
private const int EVENT_ID_AFD_CORRELATION_ID = 10; // Failed to get AFD correlation ID
private const int EVENT_ID_METRIC_BUFFER_OVERFLOW = 11; // Metric serialization buffer overflow
private const int EVENT_ID_TRACE_BUFFER_OVERFLOW = 12; // Trace serialization buffer overflow
private const int EVENT_ID_LOG_BUFFER_OVERFLOW = 13; // Log serialization buffer overflow

[NonEvent]
public void FailedToSendTraceData(Exception ex)
{
if (this.IsEnabled(EventLevel.Error, EventKeywords.All))
{
if (GenevaBufferOverflowExceptionHelper.TryReportTraceBufferOverflow(ex))
{
return;
}

// https://docs.microsoft.com/en-us/windows/win32/etw/about-event-tracing
// ETW has a size limit: The total event size is greater than 64K. This includes the ETW header plus the data or payload.
// TODO: Do not hit ETW size limit even for external library exception stack. But what is the ETW header size?
Expand All @@ -43,6 +51,11 @@ public void FailedToSendLogData(Exception ex)
{
if (this.IsEnabled(EventLevel.Error, EventKeywords.All))
{
if (GenevaBufferOverflowExceptionHelper.TryReportLogBufferOverflow(ex))
{
return;
}

// TODO: Do not hit ETW size limit even for external library exception stack.
this.FailedToSendLogData(ex.ToInvariantString());
}
Expand All @@ -53,6 +66,11 @@ public void FailedToSendMetricData(string monitoringAccount, string metricNamesp
{
if (this.IsEnabled(EventLevel.Error, EventKeywords.All))
{
if (GenevaBufferOverflowExceptionHelper.TryReportMetricBufferOverflow(metricName, ex))
{
return;
}

// TODO: Do not hit ETW size limit even for external library exception stack.
this.FailedToSendMetricData(monitoringAccount, metricNamespace, metricName, ex.ToInvariantString());
}
Expand Down Expand Up @@ -157,4 +175,22 @@ public void FailedToGetAFDCorrelationId(string error)
{
this.WriteEvent(EVENT_ID_AFD_CORRELATION_ID, error);
}

[Event(EVENT_ID_METRIC_BUFFER_OVERFLOW, Message = "Failed to export '{0}' metric: the {1}-byte serialization buffer was exceeded. Reduce the number or size of metric dimensions.", Level = EventLevel.Error)]
public void MetricSerializationBufferFull(string metricName, int bufferSizeBytes)
{
this.WriteEvent(EVENT_ID_METRIC_BUFFER_OVERFLOW, metricName, bufferSizeBytes);
}

[Event(EVENT_ID_TRACE_BUFFER_OVERFLOW, Message = "Failed to export trace data: the serialization buffer capacity was exceeded. Reduce the number or size of span attributes, events, or links.", Level = EventLevel.Error)]
public void TraceSerializationBufferFull()
{
this.WriteEvent(EVENT_ID_TRACE_BUFFER_OVERFLOW);
}

[Event(EVENT_ID_LOG_BUFFER_OVERFLOW, Message = "Failed to export log data: the serialization buffer capacity was exceeded. Reduce the size of the log body, attributes, or scopes.", Level = EventLevel.Error)]
public void LogSerializationBufferFull()
{
this.WriteEvent(EVENT_ID_LOG_BUFFER_OVERFLOW);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0

#if NET
using Microsoft.LinuxTracepoints.Provider;
#endif

namespace OpenTelemetry.Exporter.Geneva;

internal static class GenevaBufferOverflowExceptionHelper
{
internal const string MetricBufferTooSmallMessage = "The buffer is too small to write a value at the specified index.";

internal static bool TryReportTraceBufferOverflow(Exception ex)
{
if (!IsTraceBufferOverflow(ex))
{
return false;
}

ExporterEventSource.Log.TraceSerializationBufferFull();
return true;
}

internal static bool TryReportLogBufferOverflow(Exception ex)
{
if (!IsLogBufferOverflow(ex))
{
return false;
}

ExporterEventSource.Log.LogSerializationBufferFull();
return true;
}

internal static bool TryReportMetricBufferOverflow(string metricName, Exception ex)
{
if (!IsMetricBufferOverflow(ex))
{
return false;
}

ExporterEventSource.Log.MetricSerializationBufferFull(metricName, GenevaMetricExporter.BufferSize);
return true;
}

internal static bool TryReportLogBufferOverflow(int errorCode)
{
if (!IsEventHeaderBufferOverflow(errorCode))
{
return false;
}

ExporterEventSource.Log.LogSerializationBufferFull();
return true;
}

internal static bool IsTraceBufferOverflow(Exception ex) =>
IsMsgPackBufferOverflow(ex) || IsEventBuilderBufferOverflow(ex);

internal static bool IsLogBufferOverflow(Exception ex) =>
IsMsgPackBufferOverflow(ex) || IsEventBuilderBufferOverflow(ex);

internal static bool IsMetricBufferOverflow(Exception ex) =>
IsMsgPackBufferOverflow(ex) || IsMetricSerializerBufferOverflow(ex);

internal static bool IsMsgPackBufferOverflow(Exception ex) =>
ex is IndexOutOfRangeException || ex.GetType() == typeof(ArgumentException);

internal static bool IsMetricSerializerBufferOverflow(Exception ex) =>
ex is ArgumentOutOfRangeException argumentOutOfRangeException
&& argumentOutOfRangeException.Message.Contains(MetricBufferTooSmallMessage, StringComparison.Ordinal);
Comment thread
martincostello marked this conversation as resolved.

internal static bool IsEventBuilderBufferOverflow(Exception ex) =>
ex is InvalidOperationException { Message: "Event too large" };

internal static bool IsEventHeaderBufferOverflow(int errorCode) =>
#if NET
errorCode == TracepointHandle.EventTooBigError;
#else
false;
#endif
}
Original file line number Diff line number Diff line change
Expand Up @@ -467,17 +467,15 @@ public static void SerializeSpanOfBytes(byte[] buffer, ref int bufferIndex, Read

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void CheckBounds(byte[] buffer, int index, int size, [CallerArgumentExpression(nameof(index))] string? paramName = default)
#if NET
=> ArgumentOutOfRangeException.ThrowIfGreaterThan(index + size, buffer.Length, paramName);
#else
{
if (buffer.Length < index + size)
{
ThrowArgumentOutOfRange(index, paramName);
}
}

// Always throw with the shared message so the overflow can be detected consistently across
// target frameworks (ArgumentOutOfRangeException.ThrowIfGreaterThan uses a different message).
private static void ThrowArgumentOutOfRange(object value, [CallerArgumentExpression(nameof(value))] string? paramName = default)
=> throw new ArgumentOutOfRangeException(paramName, value, "The buffer is too small to write a value at the specified index.");
#endif
=> throw new ArgumentOutOfRangeException(paramName, value, GenevaBufferOverflowExceptionHelper.MetricBufferTooSmallMessage);
}
Original file line number Diff line number Diff line change
Expand Up @@ -182,9 +182,21 @@ internal static void SerializeTag(byte[] buffer, ref int cursor, string key, obj
ProtobufSerializerHelper.WriteTagAndLengthPrefix(buffer, ref anyValueTagAndLengthIndex, cursor - anyValueIndex, FieldNumberConstants.KeyValue_value, WireType.LEN);
ProtobufSerializerHelper.WriteTagAndLengthPrefix(buffer, ref keyValueTagAndLengthIndex, cursor - keyValueIndex, fieldNumber, WireType.LEN);
}
catch
catch (Exception ex) when (!GenevaBufferOverflowExceptionHelper.IsMetricBufferOverflow(ex))
{
// TODO: log exception.
// Buffer overflow exceptions are intentionally not caught here so they propagate to the
// per-metric-point handler, which reports them via the dedicated buffer overflow event.
}
}

internal void HandleMetricSerializationException(Metric metric, Exception ex)
{
this.metricExportResult = ExportResult.Failure;

if (!GenevaBufferOverflowExceptionHelper.TryReportMetricBufferOverflow(metric.Name, ex))
{
ExporterEventSource.Log.FailedToSerializeMetric(metric.Name, ex);
}
}

Expand Down Expand Up @@ -375,8 +387,7 @@ private void SerializeMetric(byte[] buffer, ref int cursor, Metric metric)
}
catch (Exception ex)
{
this.metricExportResult = ExportResult.Failure;
ExporterEventSource.Log.FailedToSerializeMetric(metric.Name, ex);
this.HandleMetricSerializationException(metric, ex);
}
}

Expand Down Expand Up @@ -415,8 +426,7 @@ private void SerializeMetric(byte[] buffer, ref int cursor, Metric metric)
}
catch (Exception ex)
{
this.metricExportResult = ExportResult.Failure;
ExporterEventSource.Log.FailedToSerializeMetric(metric.Name, ex);
this.HandleMetricSerializationException(metric, ex);
}
}

Expand Down Expand Up @@ -448,8 +458,7 @@ private void SerializeMetric(byte[] buffer, ref int cursor, Metric metric)
}
catch (Exception ex)
{
this.metricExportResult = ExportResult.Failure;
ExporterEventSource.Log.FailedToSerializeMetric(metric.Name, ex);
this.HandleMetricSerializationException(metric, ex);
}
}

Expand Down Expand Up @@ -481,8 +490,7 @@ private void SerializeMetric(byte[] buffer, ref int cursor, Metric metric)
}
catch (Exception ex)
{
this.metricExportResult = ExportResult.Failure;
ExporterEventSource.Log.FailedToSerializeMetric(metric.Name, ex);
this.HandleMetricSerializationException(metric, ex);
}
}

Expand Down Expand Up @@ -563,8 +571,7 @@ private void SerializeMetric(byte[] buffer, ref int cursor, Metric metric)
}
catch (Exception ex)
{
this.metricExportResult = ExportResult.Failure;
ExporterEventSource.Log.FailedToSerializeMetric(metric.Name, ex);
this.HandleMetricSerializationException(metric, ex);
}
}

Expand Down Expand Up @@ -656,8 +663,7 @@ private void SerializeMetric(byte[] buffer, ref int cursor, Metric metric)
}
catch (Exception ex)
{
this.metricExportResult = ExportResult.Failure;
ExporterEventSource.Log.FailedToSerializeMetric(metric.Name, ex);
this.HandleMetricSerializationException(metric, ex);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,15 +79,11 @@ internal static void WriteDoubleWithTag(byte[] buffer, ref int cursor, int field

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void WriteLengthCustom(byte[] buffer, ref int cursor, int length)
{
WriteVarintCustom(buffer, ref cursor, (uint)length);
}
=> WriteVarintCustom(buffer, ref cursor, (uint)length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void WriteLength(byte[] buffer, ref int cursor, int length)
{
WriteVarint32(buffer, ref cursor, (uint)length);
}
=> WriteVarint32(buffer, ref cursor, (uint)length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void WriteVarintCustom(byte[] buffer, ref int cursor, uint value)
Expand Down Expand Up @@ -150,9 +146,7 @@ internal static void WriteVarint64(byte[] buffer, ref int cursor, ulong value)

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void WriteTag(byte[] buffer, ref int cursor, int fieldNumber, WireType type)
{
WriteVarint32(buffer, ref cursor, GetTagValue(fieldNumber, type));
}
=> WriteVarint32(buffer, ref cursor, GetTagValue(fieldNumber, type));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void WriteTagAndLengthPrefix(byte[] buffer, ref int cursor, int contentLength, int fieldNumber, WireType type)
Expand All @@ -164,22 +158,20 @@ internal static void WriteTagAndLengthPrefix(byte[] buffer, ref int cursor, int
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void WriteFixed64LittleEndianFormat(byte[] buffer, ref int cursor, ulong value)
{
if (cursor < buffer.Length)
if (cursor + Fixed64Size > buffer.Length)
{
var span = new Span<byte>(buffer, cursor, Fixed64Size);
// Surface the overflow with the shared message instead of silently dropping the value
// so it can be detected and reported by GenevaBufferOverflowExceptionHelper.
throw new ArgumentOutOfRangeException(nameof(cursor), cursor, GenevaBufferOverflowExceptionHelper.MetricBufferTooSmallMessage);
}

BinaryPrimitives.WriteUInt64LittleEndian(span, value);
var span = new Span<byte>(buffer, cursor, Fixed64Size);

cursor += Fixed64Size;
}
else
{
// TODO: handle insufficient space.
}
BinaryPrimitives.WriteUInt64LittleEndian(span, value);

cursor += Fixed64Size;
}

internal static uint GetTagValue(int fieldNumber, WireType wireType)
{
return ((uint)(fieldNumber << 3)) | (uint)wireType;
}
=> ((uint)(fieldNumber << 3)) | (uint)wireType;
}
Loading
Loading