From 81a9cf49c5d319bca4fb91d9cf14c3d261ca6993 Mon Sep 17 00:00:00 2001 From: martincostello Date: Sat, 4 Jul 2026 10:07:58 +0100 Subject: [PATCH 1/4] [Exporter.Prometheus] Configurable buffer size - Allow the maximum buffer size to be configured to allow for scrape responses beyond 100,000 metrics. - Return an HTTP 500 if the buffer is not large enough to contain the scrape response. --- .../.publicApi/PublicAPI.Unshipped.txt | 2 + .../CHANGELOG.md | 8 +++ .../PrometheusAspNetCoreOptions.cs | 12 ++++ .../PrometheusExporterMiddleware.cs | 30 +++++---- .../.publicApi/PublicAPI.Unshipped.txt | 12 ++-- .../CHANGELOG.md | 8 +++ .../Shared/PrometheusCollectionManager.cs | 40 ++++++++++-- .../Internal/Shared/PrometheusExporter.cs | 3 + .../Shared/PrometheusExporterEventSource.cs | 4 ++ .../Shared/PrometheusExporterOptions.cs | 37 +++++++++++ .../PrometheusHttpListener.cs | 37 +++++++---- ...pListenerMeterProviderBuilderExtensions.cs | 1 + .../PrometheusHttpListenerOptions.cs | 17 +++++ .../PrometheusExporterMiddlewareTests.cs | 28 ++++++++ .../PrometheusCollectionManagerTests.cs | 64 +++++++++++++++++++ .../PrometheusHttpListenerTests.cs | 29 +++++++++ 16 files changed, 295 insertions(+), 37 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/.publicApi/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/.publicApi/PublicAPI.Unshipped.txt index cba2e2aea0a..f2a9d724e2c 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/.publicApi/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/.publicApi/PublicAPI.Unshipped.txt @@ -3,6 +3,8 @@ Microsoft.AspNetCore.Builder.PrometheusExporterEndpointRouteBuilderExtensions OpenTelemetry.Exporter.PrometheusAspNetCoreOptions OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.DisableTotalNameSuffixForCounters.get -> bool OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.DisableTotalNameSuffixForCounters.set -> void +OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.MaxScrapeResponseSizeBytes.get -> int +OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.MaxScrapeResponseSizeBytes.set -> void OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.PrometheusAspNetCoreOptions() -> void OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.ScopeInfoEnabled.get -> bool OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.ScopeInfoEnabled.set -> void diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md index c975dde0869..a2e252bfd54 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md @@ -35,6 +35,14 @@ Notes](../../RELEASENOTES.md). when negotiated via the `Accept` header. ([#7440](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7440)) +* Add `PrometheusAspNetCoreOptions.MaxScrapeResponseSizeBytes` to configure + the maximum size of a scrape response. The default is now ~166 MiB. + ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/pull/TODO)) + +* A scrape whose serialized output exceeds the maximum scrape response size + limit now responds with HTTP 500. + ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/pull/XXXX)) + ## 1.16.0-beta.1 Released 2026-Jun-10 diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusAspNetCoreOptions.cs b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusAspNetCoreOptions.cs index 5a5c75401b2..83e0eb6f117 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusAspNetCoreOptions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusAspNetCoreOptions.cs @@ -58,5 +58,17 @@ public bool TargetInfoEnabled set => this.ExporterOptions.TargetInfoEnabled = value; } + /// + /// Gets or sets the maximum size in bytes that a single scrape response is allowed to grow to. Default value: ~166 MiB. + /// + /// + /// Increase this value when exposing a very large number of time series. + /// + public int MaxScrapeResponseSizeBytes + { + get => this.ExporterOptions.MaxScrapeResponseSizeBytes; + set => this.ExporterOptions.MaxScrapeResponseSizeBytes = value; + } + internal PrometheusExporterOptions ExporterOptions { get; } = new(); } diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs index c620db5499a..bdbea89449c 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs @@ -85,21 +85,29 @@ public async Task InvokeAsync(HttpContext httpContext) { linkedCts.Token.ThrowIfCancellationRequested(); - var dataView = collectionResponse.View; - - response.StatusCode = StatusCodes.Status200OK; - - if (dataView.Count > 0) + if (!collectionResponse.Succeeded) { - response.Headers.Append("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R")); - response.ContentType = PrometheusProtocol.GetContentType(protocol); - - await WriteResponseAsync(response, dataView.Array.AsMemory(0, dataView.Count), AcceptsGZip(requestHeaders), linkedCts.Token); + PrometheusExporterEventSource.Log.ScrapeFailed(); + response.StatusCode = StatusCodes.Status500InternalServerError; } else { - // It's not expected to have no metrics to collect, but it's not necessarily a failure, either. - PrometheusExporterEventSource.Log.NoMetrics(); + var dataView = collectionResponse.View; + + response.StatusCode = StatusCodes.Status200OK; + + if (dataView.Count > 0) + { + response.Headers.Append("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R")); + response.ContentType = PrometheusProtocol.GetContentType(protocol); + + await WriteResponseAsync(response, dataView.Array.AsMemory(0, dataView.Count), AcceptsGZip(requestHeaders), linkedCts.Token); + } + else + { + // It's not expected to have no metrics to collect, but it's not necessarily a failure, either. + PrometheusExporterEventSource.Log.NoMetrics(); + } } } catch (OperationCanceledException ex) when (ex.CancellationToken == linkedCts.Token) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt index d75614e7e12..91b32ff9ab8 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt @@ -1,21 +1,23 @@ OpenTelemetry.Exporter.PrometheusHttpListenerOptions OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ConfigureHttpListener.get -> System.Action? OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ConfigureHttpListener.set -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.get -> bool +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.set -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes.get -> int +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Host.get -> string! OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Host.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Port.get -> int OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Port.set -> void -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.get -> bool -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.set -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.PrometheusHttpListenerOptions() -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScopeInfoEnabled.get -> bool OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScopeInfoEnabled.set -> void -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.TargetInfoEnabled.get -> bool -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.TargetInfoEnabled.set -> void -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.PrometheusHttpListenerOptions() -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScrapeEndpointPath.get -> string? OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScrapeEndpointPath.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScrapeResponseCacheDurationMilliseconds.get -> int OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScrapeResponseCacheDurationMilliseconds.set -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.TargetInfoEnabled.get -> bool +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.TargetInfoEnabled.set -> void OpenTelemetry.Metrics.PrometheusHttpListenerMeterProviderBuilderExtensions static OpenTelemetry.Metrics.PrometheusHttpListenerMeterProviderBuilderExtensions.AddPrometheusHttpListener(this OpenTelemetry.Metrics.MeterProviderBuilder! builder) -> OpenTelemetry.Metrics.MeterProviderBuilder! static OpenTelemetry.Metrics.PrometheusHttpListenerMeterProviderBuilderExtensions.AddPrometheusHttpListener(this OpenTelemetry.Metrics.MeterProviderBuilder! builder, string? name, System.Action? configure) -> OpenTelemetry.Metrics.MeterProviderBuilder! diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md index 5aad537d7df..631e7336396 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md @@ -41,6 +41,14 @@ Notes](../../RELEASENOTES.md). when negotiated via the `Accept` header. ([#7440](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7440)) +* Add `PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes` to configure + the maximum size of a scrape response. The default is now ~166 MiB. + ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/pull/TODO)) + +* A scrape whose serialized output exceeds the maximum scrape response size + limit now responds with HTTP 500. + ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/pull/TODO)) + ## 1.16.0-beta.1 Released 2026-Jun-10 diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs index a8f5f65e5ce..854a1b10f53 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs @@ -22,6 +22,8 @@ internal sealed class PrometheusCollectionManager private readonly long baseTimestamp = Stopwatch.GetTimestamp(); private readonly PrometheusExporter.ExportFunc onCollectRef; private readonly Dictionary metricsCache; + private readonly int maxBufferSize; + private int metricsCacheCount; private int globalLockState; private CollectionContext? collectionContext; @@ -37,6 +39,7 @@ public PrometheusCollectionManager(PrometheusExporter exporter) this.onCollectRef = this.OnCollect; this.metricsCache = []; this.GetElapsedTime = () => Stopwatch.GetElapsedTime(this.baseTimestamp); + this.maxBufferSize = this.exporter.MaxScrapeResponseSizeBytes; } internal Func UtcNow { get; set; } = static () => DateTime.UtcNow; @@ -474,8 +477,12 @@ state.GeneratedAtElapsed is { } generatedAtElapsed && } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private PrometheusProtocolState GetProtocolState(in PrometheusProtocol protocol) - => this.protocolStates.GetOrAdd(protocol, static _ => new()); + private PrometheusProtocolState GetProtocolState(in PrometheusProtocol protocol) => +#if NET + this.protocolStates.GetOrAdd(protocol, static (_, maxBufferSize) => new(maxBufferSize), this.maxBufferSize); +#else + this.protocolStates.GetOrAdd(protocol, (_) => new(this.maxBufferSize)); +#endif private int WriteTargetInfo(TextFormatSerializer serializer, PrometheusProtocolState state) { @@ -621,6 +628,7 @@ public CollectionResponse(ArraySegment view, DateTime generatedAtUtc, bool this.View = view; this.GeneratedAtUtc = generatedAtUtc; this.FromCache = fromCache; + this.Succeeded = true; } public readonly ArraySegment View { get; } @@ -628,6 +636,15 @@ public CollectionResponse(ArraySegment view, DateTime generatedAtUtc, bool public readonly DateTime GeneratedAtUtc { get; } public readonly bool FromCache { get; } + + /// + /// Gets a value indicating whether the collection that produced this response succeeded. + /// + /// + /// Used to distinguish between a successful collection that produced an empty response + /// and a failed collection that produced no response. + /// + public readonly bool Succeeded { get; } } private readonly struct CollectionResult @@ -784,11 +801,16 @@ public bool TryRegisterProtocol(in PrometheusProtocol protocol, bool hasActiveRe private sealed class PrometheusProtocolState { - private const int InitialBufferSize = 85_000; // Encourage the object to live in Large Object Heap (LOH) - private const int MaxBufferSize = 100 * 1024 * 1024; // 100 MB + private const int InitialBufferSize = PrometheusExporterOptions.InitialScrapeResponseSizeBytes; + private readonly int maxBufferSize; private int readerCount; + public PrometheusProtocolState(int maxBufferSize) + { + this.maxBufferSize = Math.Max(maxBufferSize, InitialBufferSize); + } + public static ArraySegment EmptyView { get; } = #if NET ArraySegment.Empty; @@ -815,13 +837,17 @@ public void IncrementReaderCount() public bool TryExpandBuffer() { - var newBufferSize = this.Buffer.Length * 2; - - if (newBufferSize > MaxBufferSize) + if (this.Buffer.Length >= this.maxBufferSize) { return false; } + // Grow by doubling, but never past the configured maximum. Clamping + // to the maximum (rather than refusing the grow outright when the + // doubled size would overshoot) means the entire configured budget + // is usable, with no unreachable remainder. + var newBufferSize = (int)Math.Min((long)this.Buffer.Length * 2, this.maxBufferSize); + var expanded = new byte[newBufferSize]; this.Buffer.CopyTo(expanded, 0); this.Buffer = expanded; diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporter.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporter.cs index cb21b87046b..8f8223a406e 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporter.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporter.cs @@ -27,6 +27,7 @@ public PrometheusExporter(PrometheusExporterOptions options) this.ScrapeResponseCacheDurationMilliseconds = options.ScrapeResponseCacheDurationMilliseconds; this.TargetInfoEnabled = options.TargetInfoEnabled; this.DisableTotalNameSuffixForCounters = options.DisableTotalNameSuffixForCounters; + this.MaxScrapeResponseSizeBytes = options.MaxScrapeResponseSizeBytes; this.CollectionManager = new PrometheusCollectionManager(this); } @@ -52,6 +53,8 @@ public PrometheusExporter(PrometheusExporterOptions options) internal bool DisableTotalNameSuffixForCounters { get; } + internal int MaxScrapeResponseSizeBytes { get; } + internal Resource Resource { get => field ??= this.ParentProvider.GetResource(); diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterEventSource.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterEventSource.cs index a221ae9d29b..df6c69ce490 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterEventSource.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterEventSource.cs @@ -103,4 +103,8 @@ public void MetricIgnored(Metrics.Metric metric) [Event(10, Message = "Metric '{0}' ignored as metrics of type '{1}' are not supported by Prometheus.", Level = EventLevel.Verbose)] public void MetricIgnored(string metricName, string metricType) => this.WriteEvent(10, metricName, metricType); + + [Event(11, Message = "Failed to collect metrics for a scrape request; the response may have exceeded the configured maximum size.", Level = EventLevel.Error)] + public void ScrapeFailed() + => this.WriteEvent(11); } diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterOptions.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterOptions.cs index f58c742db20..48602a56bce 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterOptions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterOptions.cs @@ -10,11 +10,31 @@ namespace OpenTelemetry.Exporter.Prometheus; /// internal sealed class PrometheusExporterOptions { + /// + /// The initial scrape response buffer size in bytes. + /// The buffer is always allocated at this size, so it is also the + /// smallest meaningful value for . + /// + public const int InitialScrapeResponseSizeBytes = 85_000; // Encourage the object to live in the Large Object Heap (LOH). + + /// + /// The default maximum scrape response size in bytes (~166 MiB). + /// + /// + /// The response buffer starts at + /// and grows on demand by doubling. This default is equal to that size + /// multiplied by 2^11, i.e. the largest size reachable by that doubling + /// sequence, so the whole budget is usable and none is left as an unreachable + /// remainder. + /// + public const int DefaultMaxScrapeResponseSizeBytes = InitialScrapeResponseSizeBytes * 2048; + public PrometheusExporterOptions() { this.ScopeInfoEnabled = true; this.ScrapeResponseCacheDurationMilliseconds = 300; this.TargetInfoEnabled = true; + this.MaxScrapeResponseSizeBytes = DefaultMaxScrapeResponseSizeBytes; } /// @@ -49,4 +69,21 @@ public int ScrapeResponseCacheDurationMilliseconds /// Gets or sets a value indicating whether addition of _total suffix for counter metric names is disabled. Default value: . /// public bool DisableTotalNameSuffixForCounters { get; set; } + + /// + /// Gets or sets the maximum size in bytes that a single scrape response is + /// allowed to grow to. Default value: (~166 MiB). + /// + /// + /// Increase this value when exposing a very large number of time series. + /// + public int MaxScrapeResponseSizeBytes + { + get; + set + { + Guard.ThrowIfOutOfRange(value, min: InitialScrapeResponseSizeBytes); + field = value; + } + } } diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs index 9748480224b..c4e45d91c1d 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs @@ -262,25 +262,34 @@ private async Task ProcessRequestAsync(HttpListenerContext context, Cancellation context.Response.Headers.Add("Server", string.Empty); - var dataView = collectionResponse.View; - - if (dataView.Count > 0) + if (!collectionResponse.Succeeded) { - context.Response.StatusCode = 200; - context.Response.Headers.Add("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R")); - context.Response.ContentType = PrometheusProtocol.GetContentType(protocol); + PrometheusExporterEventSource.Log.ScrapeFailed(); + context.Response.StatusCode = 500; + context.Response.ContentLength64 = 0; + } + else + { + var dataView = collectionResponse.View; + + if (dataView.Count > 0) + { + context.Response.StatusCode = 200; + context.Response.Headers.Add("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R")); + context.Response.ContentType = PrometheusProtocol.GetContentType(protocol); #if NET - await context.Response.OutputStream.WriteAsync(dataView.Array.AsMemory(0, dataView.Count), linkedCts.Token).ConfigureAwait(false); + await context.Response.OutputStream.WriteAsync(dataView.Array.AsMemory(0, dataView.Count), linkedCts.Token).ConfigureAwait(false); #else - await context.Response.OutputStream.WriteAsync(dataView.Array, 0, dataView.Count, linkedCts.Token).ConfigureAwait(false); + await context.Response.OutputStream.WriteAsync(dataView.Array, 0, dataView.Count, linkedCts.Token).ConfigureAwait(false); #endif - } - else - { - // It's not expected to have no metrics to collect, but it's not necessarily a failure, either. - context.Response.StatusCode = 200; - PrometheusExporterEventSource.Log.NoMetrics(); + } + else + { + // It's not expected to have no metrics to collect, but it's not necessarily a failure, either. + context.Response.StatusCode = 200; + PrometheusExporterEventSource.Log.NoMetrics(); + } } } catch (OperationCanceledException ex) when (ex.CancellationToken == requestCancelled.Token) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerMeterProviderBuilderExtensions.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerMeterProviderBuilderExtensions.cs index 091a910d034..1c10fa28ee4 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerMeterProviderBuilderExtensions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerMeterProviderBuilderExtensions.cs @@ -71,6 +71,7 @@ private static BaseExportingMetricReader BuildPrometheusHttpListenerMetricReader ScrapeResponseCacheDurationMilliseconds = options.ScrapeResponseCacheDurationMilliseconds, TargetInfoEnabled = options.TargetInfoEnabled, DisableTotalNameSuffixForCounters = options.DisableTotalNameSuffixForCounters, + MaxScrapeResponseSizeBytes = options.MaxScrapeResponseSizeBytes, }); var reader = new BaseExportingMetricReader(exporter) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerOptions.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerOptions.cs index 30ce6d15d51..98dd1f43d85 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerOptions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerOptions.cs @@ -49,6 +49,7 @@ internal PrometheusHttpListenerOptions(IConfiguration configuration) this.Host = host; this.Port = port; this.ScrapeResponseCacheDurationMilliseconds = 300; + this.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.DefaultMaxScrapeResponseSizeBytes; } /// @@ -99,6 +100,22 @@ public int ScrapeResponseCacheDurationMilliseconds /// public bool TargetInfoEnabled { get; set; } = true; + /// + /// Gets or sets the maximum size in bytes that a single scrape response is allowed to grow to. Default value: ~166 MiB. + /// + /// + /// Increase this value when exposing a very large number of time series. + /// + public int MaxScrapeResponseSizeBytes + { + get; + set + { + Guard.ThrowIfOutOfRange(value, min: PrometheusExporterOptions.InitialScrapeResponseSizeBytes); + field = value; + } + } + /// /// Gets or sets an optional callback to apply custom configuration for the /// instance used by the exporter. diff --git a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs index d9ffec481ce..46318563dad 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs @@ -428,6 +428,34 @@ public async Task BufferSizeIncreasesWithLotOfMetrics() await Verify(output, "text", PrometheusSerializerTests.VerifySettings); } + [Fact] + public async Task ScrapeExceedingMaxResponseSizeReturns500() + { + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + configureOptions: o => o.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.InitialScrapeResponseSizeBytes); + + using var meter = new Meter(MeterName, MeterVersion); + + // Emit enough series that the serialized response far exceeds the configured maximum, so + // the response buffer cannot grow to hold it and the scrape fails rather than returning a + // misleading empty 200 response. + for (var x = 0; x < 2_000; x++) + { + meter.CreateCounter("counter_double_" + x, unit: "By").Add(1); + } + + host.Services.GetRequiredService().ForceFlush(); + + using var client = host.GetTestClient(); + + using var response = await client.GetAsync(new Uri("/metrics", UriKind.Relative)); + + Assert.Equal(HttpStatusCode.InternalServerError, response.StatusCode); + + await host.StopAsync(); + } + [Fact] public async Task InvokeAsync_WhenNoData_Returns200() { diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs index c20ad4926ec..d226e984a88 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs @@ -877,6 +877,70 @@ public async Task LargeScrapeExpandsBufferAndReturnsCompleteResponse(bool openMe } } + [Theory] + [InlineData(85_000, false)] // The buffer cannot grow beyond its initial size, so the large scrape is dropped. + [InlineData(64 * 1024 * 1024, true)] // Ample budget, so the large scrape is served in full. + public async Task MaxScrapeResponseSizeBytesBoundsTheResponseBuffer(int maxScrapeResponseSizeBytes, bool expectNonEmpty) + { + using var meter = CreateMeter(); + + using var provider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) +#if PROMETHEUS_HTTP_LISTENER + .AddPrometheusHttpListener(options => + { + options.MaxScrapeResponseSizeBytes = maxScrapeResponseSizeBytes; + options.ScrapeResponseCacheDurationMilliseconds = 0; + }) +#elif PROMETHEUS_ASPNETCORE + .AddPrometheusExporter(options => + { + options.MaxScrapeResponseSizeBytes = maxScrapeResponseSizeBytes; + options.ScrapeResponseCacheDurationMilliseconds = 0; + }) +#endif + .Build(); + +#pragma warning disable CA2000 // MeterProvider owns exporter lifecycle + Assert.True(provider.TryFindExporter(out PrometheusExporter? exporter)); +#pragma warning restore CA2000 // MeterProvider owns exporter lifecycle + + // Emit enough unique time series that the serialized output is far larger than the + // 85,000-byte initial scrape buffer, so serving the scrape requires the buffer to grow. + var counter = meter.CreateCounter("test_counter", description: "Help text."); + const int SeriesCount = 4_000; + for (var i = 0; i < SeriesCount; i++) + { + counter.Add( + 1234567890123456789L, + new KeyValuePair("index", $"series-value-{i:D8}-padding-padding-padding")); + } + + var protocol = GetProtocol(openMetricsRequested: false); + var response = await exporter!.CollectionManager.EnterCollect(protocol); + + try + { + if (expectNonEmpty) + { + // The configured budget is large enough for the buffer to grow and serve the scrape. + Assert.True(response.Succeeded, "Expected the collection to succeed."); + Assert.True(response.View.Count > 85_000, $"Expected a complete response, but only {response.View.Count} bytes were written."); + } + else + { + // The buffer cannot grow past the configured maximum, so the scrape is dropped and a + // failed (empty) response is returned rather than allocating without bound. + Assert.False(response.Succeeded, "Expected the collection to fail."); + Assert.Equal(0, response.View.Count); + } + } + finally + { + exporter.CollectionManager.ExitCollect(protocol); + } + } + private static int CountOccurrences(string value, string substring) { var count = 0; diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs index c9978f70ec7..6d540dc9585 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs @@ -490,6 +490,35 @@ public async Task WhenRequestDeadlineInvalid_Returns200(string scrapeTimeoutSeco Assert.Equal(HttpStatusCode.OK, response.StatusCode); } + [Fact] + public async Task WhenResponseExceedsMaxScrapeResponseSize_Returns500() + { + using var meter = new Meter(MeterName, MeterVersion); + + using var context = CreateMeterProvider( + meter, + configureListener: options => + { + options.Port = GetRandomPort(); + options.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.InitialScrapeResponseSizeBytes; + return options.Port; + }); + + // Emit enough series that the serialized response far exceeds the configured maximum, so + // the response buffer cannot grow to hold it and the scrape fails rather than returning a + // misleading empty 200 response. + for (var x = 0; x < 2_000; x++) + { + meter.CreateCounter("counter_double_" + x, unit: "By").Add(1); + } + + using var client = new HttpClient { BaseAddress = context.BaseAddress }; + + using var response = await client.GetAsync(new Uri("metrics", UriKind.Relative)); + + Assert.Equal(HttpStatusCode.InternalServerError, response.StatusCode); + } + internal static MeterProviderTestContext CreateMeterProvider( Meter meter, Func? configureListener = null, From ca8859a5567cf80fc3f697b1aa27ba1865267c05 Mon Sep 17 00:00:00 2001 From: Martin Costello Date: Sat, 4 Jul 2026 10:42:59 +0100 Subject: [PATCH 2/4] [Exporter.Prometheus] Update CHANGELOGs Add PR number. --- src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md | 4 ++-- .../CHANGELOG.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md index a2e252bfd54..c3ac5c1d3c1 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md @@ -37,11 +37,11 @@ Notes](../../RELEASENOTES.md). * Add `PrometheusAspNetCoreOptions.MaxScrapeResponseSizeBytes` to configure the maximum size of a scrape response. The default is now ~166 MiB. - ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/pull/TODO)) + ([#7487](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7487)) * A scrape whose serialized output exceeds the maximum scrape response size limit now responds with HTTP 500. - ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/pull/XXXX)) + ([#7487](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7487)) ## 1.16.0-beta.1 diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md index 631e7336396..1c3f7132322 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md @@ -43,11 +43,11 @@ Notes](../../RELEASENOTES.md). * Add `PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes` to configure the maximum size of a scrape response. The default is now ~166 MiB. - ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/pull/TODO)) + ([#7487](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7487)) * A scrape whose serialized output exceeds the maximum scrape response size limit now responds with HTTP 500. - ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/pull/TODO)) + ([#7487](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7487)) ## 1.16.0-beta.1 From cc9c6e81b4392571897dcd5a4a409135cbd7a0d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kie=C5=82kowicz?= Date: Tue, 7 Jul 2026 09:22:54 +0200 Subject: [PATCH 3/4] [Exporter.Prometheus] Share failed scrape results Record an explicit failed response for every protocol whose serialization was attempted but did not succeed. This lets callers that joined the same in-flight collection observe a terminal buffer overflow instead of interpreting an absent response as a transient failure and repeating the entire scrape. Keep responses absent when collection fails before serialization so the existing retry behavior for transient collection failures remains unchanged. Add concurrent regression coverage for both Prometheus exporters to verify that joined oversized scrapes share one collection and both receive failure responses. Assisted-by: OpenAI Codex --- .../Shared/PrometheusCollectionManager.cs | 9 ++ .../PrometheusCollectionManagerTests.cs | 98 ++++++++++++++++++- 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs index 854a1b10f53..d9c1929adf0 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs @@ -427,6 +427,14 @@ private CollectionResult CreateCollectionResult(CollectionContext collectionCont var protocols = executionResult.Protocols ?? collectionContext.FreezeProtocols(); var responses = new Dictionary(protocols.Length); + if (!succeeded && executionResult.Protocols is not null) + { + foreach (var protocol in protocols) + { + responses[protocol] = default; + } + } + if (succeeded) { var generatedAt = this.UtcNow(); @@ -438,6 +446,7 @@ private CollectionResult CreateCollectionResult(CollectionContext collectionCont if (successfulProtocols is not null && !successfulProtocols.Contains(protocol)) { + responses[protocol] = default; continue; } diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs index d226e984a88..9c53c1b1acd 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs @@ -494,6 +494,99 @@ public async Task EnterCollectRetriesAfterFailedSharedCollection() Assert.True(secondResponse.View.Count > 0); } + [Fact] + public async Task EnterCollectSharesFailedSerializationResult() + { + using var meter = CreateMeter(); +#if PROMETHEUS_HTTP_LISTENER + using var provider = CreateMeterProviderWithRandomPort( + meter, + options => options.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.InitialScrapeResponseSizeBytes); +#elif PROMETHEUS_ASPNETCORE + using var provider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) + .AddPrometheusExporter(options => + { + options.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.InitialScrapeResponseSizeBytes; + options.ScrapeResponseCacheDurationMilliseconds = 0; + }) + .Build(); +#endif + +#pragma warning disable CA2000 // MeterProvider owns exporter lifecycle + Assert.True(provider.TryFindExporter(out PrometheusExporter? exporter)); +#pragma warning restore CA2000 // MeterProvider owns exporter lifecycle + + var collectCount = 0; + var firstCollectStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allowFirstCollectToComplete = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var originalCollect = exporter!.Collect; + exporter.Collect = (timeout) => + { + var currentCollectCount = Interlocked.Increment(ref collectCount); + + if (currentCollectCount == 1) + { + firstCollectStarted.SetResult(true); + Assert.True(allowFirstCollectToComplete.Task.Wait(TimeSpan.FromSeconds(5)), "First collection did not resume."); + } + + return originalCollect!(timeout); + }; + + var counter = meter.CreateCounter("test_counter", description: "Help text."); + const int SeriesCount = 4_000; + for (var i = 0; i < SeriesCount; i++) + { + counter.Add( + 1234567890123456789L, + new KeyValuePair("index", $"series-value-{i:D8}-padding-padding-padding")); + } + + var protocol = GetProtocol(openMetricsRequested: false); + + async Task CollectAsync() + { + var response = await EnterCollectAsync(exporter, protocol); + try + { + return response; + } + finally + { + exporter.CollectionManager.ExitCollect(protocol); + } + } + + var firstCollectTask = Task.Run(CollectAsync); + + await firstCollectStarted.Task; + + var secondCollectTask = CollectAsync(); + + Assert.False(secondCollectTask.IsCompleted, "Second collection did not join the active collection."); + + allowFirstCollectToComplete.SetResult(true); + + var timeout = TimeSpan.FromSeconds(5); + var all = Task.WhenAll(firstCollectTask, secondCollectTask); + + using (var cts = new CancellationTokenSource(timeout)) + { + var completion = await Task.WhenAny(all, Task.Delay(timeout, cts.Token)); + Assert.Same(all, completion); + } + + var responses = await all; + + Assert.Equal(1, collectCount); + Assert.All(responses, response => + { + Assert.False(response.Succeeded, "Expected the collection to fail."); + Assert.Equal(0, response.View.Count); + }); + } + [Fact] public async Task OpenMetricsDoesNotEmitScopeInfoMetricFamily() { @@ -971,7 +1064,9 @@ private static int CountOccurrences(string value, string substring) private static Meter CreateMeter([CallerMemberName] string name = "") => new(name); #if PROMETHEUS_HTTP_LISTENER - private static MeterProvider CreateMeterProviderWithRandomPort(Meter meter) + private static MeterProvider CreateMeterProviderWithRandomPort( + Meter meter, + Action? configure = null) { var retryAttempts = 5; @@ -987,6 +1082,7 @@ private static MeterProvider CreateMeterProviderWithRandomPort(Meter meter) { options.Port = port; options.ScrapeResponseCacheDurationMilliseconds = 0; + configure?.Invoke(options); }) .Build(); } From 354dc37f606e744c9209ee000ab279b0e0c2e662 Mon Sep 17 00:00:00 2001 From: martincostello Date: Tue, 7 Jul 2026 08:54:18 +0100 Subject: [PATCH 4/4] [Exporter.Prometheus] Sort properties Move the new property to the right sort location. --- .../.publicApi/PublicAPI.Unshipped.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt index 091791fcd16..8defa35d2f7 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt @@ -3,10 +3,10 @@ OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ConfigureHttpListener.get - OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ConfigureHttpListener.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.get -> bool OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.set -> void -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes.get -> int -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Host.get -> string! OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Host.set -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes.get -> int +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Port.get -> int OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Port.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.PrometheusHttpListenerOptions() -> void