diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/.publicApi/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/.publicApi/PublicAPI.Unshipped.txt index 9e08604f954..06fdd595788 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/.publicApi/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/.publicApi/PublicAPI.Unshipped.txt @@ -3,6 +3,8 @@ Microsoft.AspNetCore.Builder.PrometheusExporterEndpointRouteBuilderExtensions OpenTelemetry.Exporter.PrometheusAspNetCoreOptions OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.DisableTotalNameSuffixForCounters.get -> bool OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.DisableTotalNameSuffixForCounters.set -> void +OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.MaxScrapeResponseSizeBytes.get -> int +OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.MaxScrapeResponseSizeBytes.set -> void OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.PrometheusAspNetCoreOptions() -> void OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.ResourceConstantLabels.get -> System.Func? OpenTelemetry.Exporter.PrometheusAspNetCoreOptions.ResourceConstantLabels.set -> void diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md index 87c0a0a3f55..8f8c34382d9 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md @@ -40,6 +40,14 @@ Notes](../../RELEASENOTES.md). `null` (no resource attributes are added as metric labels). ([#7471](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7471)) +* Add `PrometheusAspNetCoreOptions.MaxScrapeResponseSizeBytes` to configure + the maximum size of a scrape response. The default is now ~166 MiB. + ([#7487](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7487)) + +* A scrape whose serialized output exceeds the maximum scrape response size + limit now responds with HTTP 500. + ([#7487](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7487)) + ## 1.16.0-beta.1 Released 2026-Jun-10 diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusAspNetCoreOptions.cs b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusAspNetCoreOptions.cs index 2a1a9f12bac..c44abe01e47 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusAspNetCoreOptions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusAspNetCoreOptions.cs @@ -73,5 +73,17 @@ public Func? ResourceConstantLabels set => this.ExporterOptions.ResourceConstantLabels = value; } + /// + /// Gets or sets the maximum size in bytes that a single scrape response is allowed to grow to. Default value: ~166 MiB. + /// + /// + /// Increase this value when exposing a very large number of time series. + /// + public int MaxScrapeResponseSizeBytes + { + get => this.ExporterOptions.MaxScrapeResponseSizeBytes; + set => this.ExporterOptions.MaxScrapeResponseSizeBytes = value; + } + internal PrometheusExporterOptions ExporterOptions { get; } = new(); } diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs index c620db5499a..bdbea89449c 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs @@ -85,21 +85,29 @@ public async Task InvokeAsync(HttpContext httpContext) { linkedCts.Token.ThrowIfCancellationRequested(); - var dataView = collectionResponse.View; - - response.StatusCode = StatusCodes.Status200OK; - - if (dataView.Count > 0) + if (!collectionResponse.Succeeded) { - response.Headers.Append("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R")); - response.ContentType = PrometheusProtocol.GetContentType(protocol); - - await WriteResponseAsync(response, dataView.Array.AsMemory(0, dataView.Count), AcceptsGZip(requestHeaders), linkedCts.Token); + PrometheusExporterEventSource.Log.ScrapeFailed(); + response.StatusCode = StatusCodes.Status500InternalServerError; } else { - // It's not expected to have no metrics to collect, but it's not necessarily a failure, either. - PrometheusExporterEventSource.Log.NoMetrics(); + var dataView = collectionResponse.View; + + response.StatusCode = StatusCodes.Status200OK; + + if (dataView.Count > 0) + { + response.Headers.Append("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R")); + response.ContentType = PrometheusProtocol.GetContentType(protocol); + + await WriteResponseAsync(response, dataView.Array.AsMemory(0, dataView.Count), AcceptsGZip(requestHeaders), linkedCts.Token); + } + else + { + // It's not expected to have no metrics to collect, but it's not necessarily a failure, either. + PrometheusExporterEventSource.Log.NoMetrics(); + } } } catch (OperationCanceledException ex) when (ex.CancellationToken == linkedCts.Token) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt index 3c1097b0df7..8defa35d2f7 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/.publicApi/PublicAPI.Unshipped.txt @@ -1,23 +1,25 @@ OpenTelemetry.Exporter.PrometheusHttpListenerOptions OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ConfigureHttpListener.get -> System.Action? OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ConfigureHttpListener.set -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.get -> bool +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Host.get -> string! OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Host.set -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes.get -> int +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Port.get -> int OpenTelemetry.Exporter.PrometheusHttpListenerOptions.Port.set -> void -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.get -> bool -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.DisableTotalNameSuffixForCounters.set -> void -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScopeInfoEnabled.get -> bool -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScopeInfoEnabled.set -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.PrometheusHttpListenerOptions() -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ResourceConstantLabels.get -> System.Func? OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ResourceConstantLabels.set -> void -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.TargetInfoEnabled.get -> bool -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.TargetInfoEnabled.set -> void -OpenTelemetry.Exporter.PrometheusHttpListenerOptions.PrometheusHttpListenerOptions() -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScopeInfoEnabled.get -> bool +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScopeInfoEnabled.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScrapeEndpointPath.get -> string? OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScrapeEndpointPath.set -> void OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScrapeResponseCacheDurationMilliseconds.get -> int OpenTelemetry.Exporter.PrometheusHttpListenerOptions.ScrapeResponseCacheDurationMilliseconds.set -> void +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.TargetInfoEnabled.get -> bool +OpenTelemetry.Exporter.PrometheusHttpListenerOptions.TargetInfoEnabled.set -> void OpenTelemetry.Metrics.PrometheusHttpListenerMeterProviderBuilderExtensions static OpenTelemetry.Metrics.PrometheusHttpListenerMeterProviderBuilderExtensions.AddPrometheusHttpListener(this OpenTelemetry.Metrics.MeterProviderBuilder! builder) -> OpenTelemetry.Metrics.MeterProviderBuilder! static OpenTelemetry.Metrics.PrometheusHttpListenerMeterProviderBuilderExtensions.AddPrometheusHttpListener(this OpenTelemetry.Metrics.MeterProviderBuilder! builder, string? name, System.Action? configure) -> OpenTelemetry.Metrics.MeterProviderBuilder! diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md index c5dbddd30fa..8a5139dbd8a 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md @@ -46,6 +46,14 @@ Notes](../../RELEASENOTES.md). `null` (no resource attributes are added as metric labels). ([#7471](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7471)) +* Add `PrometheusHttpListenerOptions.MaxScrapeResponseSizeBytes` to configure + the maximum size of a scrape response. The default is now ~166 MiB. + ([#7487](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7487)) + +* A scrape whose serialized output exceeds the maximum scrape response size + limit now responds with HTTP 500. + ([#7487](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7487)) + ## 1.16.0-beta.1 Released 2026-Jun-10 diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs index be2d07bd8ad..bf20fe3978f 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs @@ -23,6 +23,8 @@ internal sealed class PrometheusCollectionManager private readonly long baseTimestamp = Stopwatch.GetTimestamp(); private readonly PrometheusExporter.ExportFunc onCollectRef; private readonly Dictionary metricsCache; + private readonly int maxBufferSize; + private int metricsCacheCount; private IReadOnlyList>? resourceConstantLabels; private bool resourceConstantLabelsComputed; @@ -41,6 +43,7 @@ public PrometheusCollectionManager(PrometheusExporter exporter) this.onCollectRef = this.OnCollect; this.metricsCache = []; this.GetElapsedTime = () => Stopwatch.GetElapsedTime(this.baseTimestamp); + this.maxBufferSize = this.exporter.MaxScrapeResponseSizeBytes; } internal Func UtcNow { get; set; } = static () => DateTime.UtcNow; @@ -430,6 +433,14 @@ private CollectionResult CreateCollectionResult(CollectionContext collectionCont var protocols = executionResult.Protocols ?? collectionContext.FreezeProtocols(); var responses = new Dictionary(protocols.Length); + if (!succeeded && executionResult.Protocols is not null) + { + foreach (var protocol in protocols) + { + responses[protocol] = default; + } + } + if (succeeded) { var generatedAt = this.UtcNow(); @@ -441,6 +452,7 @@ private CollectionResult CreateCollectionResult(CollectionContext collectionCont if (successfulProtocols is not null && !successfulProtocols.Contains(protocol)) { + responses[protocol] = default; continue; } @@ -480,8 +492,12 @@ state.GeneratedAtElapsed is { } generatedAtElapsed && } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private PrometheusProtocolState GetProtocolState(in PrometheusProtocol protocol) - => this.protocolStates.GetOrAdd(protocol, static _ => new()); + private PrometheusProtocolState GetProtocolState(in PrometheusProtocol protocol) => +#if NET + this.protocolStates.GetOrAdd(protocol, static (_, maxBufferSize) => new(maxBufferSize), this.maxBufferSize); +#else + this.protocolStates.GetOrAdd(protocol, (_) => new(this.maxBufferSize)); +#endif private IReadOnlyList>? GetResourceConstantLabels() { @@ -653,6 +669,7 @@ public CollectionResponse(ArraySegment view, DateTime generatedAtUtc, bool this.View = view; this.GeneratedAtUtc = generatedAtUtc; this.FromCache = fromCache; + this.Succeeded = true; } public readonly ArraySegment View { get; } @@ -660,6 +677,15 @@ public CollectionResponse(ArraySegment view, DateTime generatedAtUtc, bool public readonly DateTime GeneratedAtUtc { get; } public readonly bool FromCache { get; } + + /// + /// Gets a value indicating whether the collection that produced this response succeeded. + /// + /// + /// Used to distinguish between a successful collection that produced an empty response + /// and a failed collection that produced no response. + /// + public readonly bool Succeeded { get; } } private readonly struct CollectionResult @@ -816,11 +842,16 @@ public bool TryRegisterProtocol(in PrometheusProtocol protocol, bool hasActiveRe private sealed class PrometheusProtocolState { - private const int InitialBufferSize = 85_000; // Encourage the object to live in Large Object Heap (LOH) - private const int MaxBufferSize = 100 * 1024 * 1024; // 100 MB + private const int InitialBufferSize = PrometheusExporterOptions.InitialScrapeResponseSizeBytes; + private readonly int maxBufferSize; private int readerCount; + public PrometheusProtocolState(int maxBufferSize) + { + this.maxBufferSize = Math.Max(maxBufferSize, InitialBufferSize); + } + public static ArraySegment EmptyView { get; } = #if NET ArraySegment.Empty; @@ -847,13 +878,17 @@ public void IncrementReaderCount() public bool TryExpandBuffer() { - var newBufferSize = this.Buffer.Length * 2; - - if (newBufferSize > MaxBufferSize) + if (this.Buffer.Length >= this.maxBufferSize) { return false; } + // Grow by doubling, but never past the configured maximum. Clamping + // to the maximum (rather than refusing the grow outright when the + // doubled size would overshoot) means the entire configured budget + // is usable, with no unreachable remainder. + var newBufferSize = (int)Math.Min((long)this.Buffer.Length * 2, this.maxBufferSize); + var expanded = new byte[newBufferSize]; this.Buffer.CopyTo(expanded, 0); this.Buffer = expanded; diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporter.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporter.cs index 247b967b62c..982542b47b6 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporter.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporter.cs @@ -28,6 +28,7 @@ public PrometheusExporter(PrometheusExporterOptions options) this.TargetInfoEnabled = options.TargetInfoEnabled; this.DisableTotalNameSuffixForCounters = options.DisableTotalNameSuffixForCounters; this.ResourceConstantLabels = options.ResourceConstantLabels; + this.MaxScrapeResponseSizeBytes = options.MaxScrapeResponseSizeBytes; this.CollectionManager = new PrometheusCollectionManager(this); } @@ -55,6 +56,8 @@ public PrometheusExporter(PrometheusExporterOptions options) internal Func? ResourceConstantLabels { get; } + internal int MaxScrapeResponseSizeBytes { get; } + internal Resource Resource { get => field ??= this.ParentProvider.GetResource(); diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterEventSource.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterEventSource.cs index a221ae9d29b..df6c69ce490 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterEventSource.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterEventSource.cs @@ -103,4 +103,8 @@ public void MetricIgnored(Metrics.Metric metric) [Event(10, Message = "Metric '{0}' ignored as metrics of type '{1}' are not supported by Prometheus.", Level = EventLevel.Verbose)] public void MetricIgnored(string metricName, string metricType) => this.WriteEvent(10, metricName, metricType); + + [Event(11, Message = "Failed to collect metrics for a scrape request; the response may have exceeded the configured maximum size.", Level = EventLevel.Error)] + public void ScrapeFailed() + => this.WriteEvent(11); } diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterOptions.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterOptions.cs index 392dbbbd620..47c203eb70f 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterOptions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusExporterOptions.cs @@ -10,11 +10,31 @@ namespace OpenTelemetry.Exporter.Prometheus; /// internal sealed class PrometheusExporterOptions { + /// + /// The initial scrape response buffer size in bytes. + /// The buffer is always allocated at this size, so it is also the + /// smallest meaningful value for . + /// + public const int InitialScrapeResponseSizeBytes = 85_000; // Encourage the object to live in the Large Object Heap (LOH). + + /// + /// The default maximum scrape response size in bytes (~166 MiB). + /// + /// + /// The response buffer starts at + /// and grows on demand by doubling. This default is equal to that size + /// multiplied by 2^11, i.e. the largest size reachable by that doubling + /// sequence, so the whole budget is usable and none is left as an unreachable + /// remainder. + /// + public const int DefaultMaxScrapeResponseSizeBytes = InitialScrapeResponseSizeBytes * 2048; + public PrometheusExporterOptions() { this.ScopeInfoEnabled = true; this.ScrapeResponseCacheDurationMilliseconds = 300; this.TargetInfoEnabled = true; + this.MaxScrapeResponseSizeBytes = DefaultMaxScrapeResponseSizeBytes; } /// @@ -56,4 +76,21 @@ public int ScrapeResponseCacheDurationMilliseconds /// attribute. Default value: (no resource attributes are added as metric labels). /// public Func? ResourceConstantLabels { get; set; } + + /// + /// Gets or sets the maximum size in bytes that a single scrape response is + /// allowed to grow to. Default value: (~166 MiB). + /// + /// + /// Increase this value when exposing a very large number of time series. + /// + public int MaxScrapeResponseSizeBytes + { + get; + set + { + Guard.ThrowIfOutOfRange(value, min: InitialScrapeResponseSizeBytes); + field = value; + } + } } diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs index 9748480224b..c4e45d91c1d 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs @@ -262,25 +262,34 @@ private async Task ProcessRequestAsync(HttpListenerContext context, Cancellation context.Response.Headers.Add("Server", string.Empty); - var dataView = collectionResponse.View; - - if (dataView.Count > 0) + if (!collectionResponse.Succeeded) { - context.Response.StatusCode = 200; - context.Response.Headers.Add("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R")); - context.Response.ContentType = PrometheusProtocol.GetContentType(protocol); + PrometheusExporterEventSource.Log.ScrapeFailed(); + context.Response.StatusCode = 500; + context.Response.ContentLength64 = 0; + } + else + { + var dataView = collectionResponse.View; + + if (dataView.Count > 0) + { + context.Response.StatusCode = 200; + context.Response.Headers.Add("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R")); + context.Response.ContentType = PrometheusProtocol.GetContentType(protocol); #if NET - await context.Response.OutputStream.WriteAsync(dataView.Array.AsMemory(0, dataView.Count), linkedCts.Token).ConfigureAwait(false); + await context.Response.OutputStream.WriteAsync(dataView.Array.AsMemory(0, dataView.Count), linkedCts.Token).ConfigureAwait(false); #else - await context.Response.OutputStream.WriteAsync(dataView.Array, 0, dataView.Count, linkedCts.Token).ConfigureAwait(false); + await context.Response.OutputStream.WriteAsync(dataView.Array, 0, dataView.Count, linkedCts.Token).ConfigureAwait(false); #endif - } - else - { - // It's not expected to have no metrics to collect, but it's not necessarily a failure, either. - context.Response.StatusCode = 200; - PrometheusExporterEventSource.Log.NoMetrics(); + } + else + { + // It's not expected to have no metrics to collect, but it's not necessarily a failure, either. + context.Response.StatusCode = 200; + PrometheusExporterEventSource.Log.NoMetrics(); + } } } catch (OperationCanceledException ex) when (ex.CancellationToken == requestCancelled.Token) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerMeterProviderBuilderExtensions.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerMeterProviderBuilderExtensions.cs index 073f6887341..ba2305db0c2 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerMeterProviderBuilderExtensions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerMeterProviderBuilderExtensions.cs @@ -72,6 +72,7 @@ private static BaseExportingMetricReader BuildPrometheusHttpListenerMetricReader TargetInfoEnabled = options.TargetInfoEnabled, DisableTotalNameSuffixForCounters = options.DisableTotalNameSuffixForCounters, ResourceConstantLabels = options.ResourceConstantLabels, + MaxScrapeResponseSizeBytes = options.MaxScrapeResponseSizeBytes, }); var reader = new BaseExportingMetricReader(exporter) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerOptions.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerOptions.cs index 9a85daf916c..e06ce1a6b74 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerOptions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListenerOptions.cs @@ -49,6 +49,7 @@ internal PrometheusHttpListenerOptions(IConfiguration configuration) this.Host = host; this.Port = port; this.ScrapeResponseCacheDurationMilliseconds = 300; + this.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.DefaultMaxScrapeResponseSizeBytes; } /// @@ -110,6 +111,22 @@ public int ScrapeResponseCacheDurationMilliseconds /// public Func? ResourceConstantLabels { get; set; } + /// + /// Gets or sets the maximum size in bytes that a single scrape response is allowed to grow to. Default value: ~166 MiB. + /// + /// + /// Increase this value when exposing a very large number of time series. + /// + public int MaxScrapeResponseSizeBytes + { + get; + set + { + Guard.ThrowIfOutOfRange(value, min: PrometheusExporterOptions.InitialScrapeResponseSizeBytes); + field = value; + } + } + /// /// Gets or sets an optional callback to apply custom configuration for the /// instance used by the exporter. diff --git a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs index 7583caad567..cb7eec56960 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs @@ -448,6 +448,34 @@ public async Task BufferSizeIncreasesWithLotOfMetrics() await Verify(output, "text", PrometheusSerializerTests.VerifySettings); } + [Fact] + public async Task ScrapeExceedingMaxResponseSizeReturns500() + { + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + configureOptions: o => o.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.InitialScrapeResponseSizeBytes); + + using var meter = new Meter(MeterName, MeterVersion); + + // Emit enough series that the serialized response far exceeds the configured maximum, so + // the response buffer cannot grow to hold it and the scrape fails rather than returning a + // misleading empty 200 response. + for (var x = 0; x < 2_000; x++) + { + meter.CreateCounter("counter_double_" + x, unit: "By").Add(1); + } + + host.Services.GetRequiredService().ForceFlush(); + + using var client = host.GetTestClient(); + + using var response = await client.GetAsync(new Uri("/metrics", UriKind.Relative)); + + Assert.Equal(HttpStatusCode.InternalServerError, response.StatusCode); + + await host.StopAsync(); + } + [Fact] public async Task InvokeAsync_WhenNoData_Returns200() { diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs index c20ad4926ec..9c53c1b1acd 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs @@ -494,6 +494,99 @@ public async Task EnterCollectRetriesAfterFailedSharedCollection() Assert.True(secondResponse.View.Count > 0); } + [Fact] + public async Task EnterCollectSharesFailedSerializationResult() + { + using var meter = CreateMeter(); +#if PROMETHEUS_HTTP_LISTENER + using var provider = CreateMeterProviderWithRandomPort( + meter, + options => options.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.InitialScrapeResponseSizeBytes); +#elif PROMETHEUS_ASPNETCORE + using var provider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) + .AddPrometheusExporter(options => + { + options.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.InitialScrapeResponseSizeBytes; + options.ScrapeResponseCacheDurationMilliseconds = 0; + }) + .Build(); +#endif + +#pragma warning disable CA2000 // MeterProvider owns exporter lifecycle + Assert.True(provider.TryFindExporter(out PrometheusExporter? exporter)); +#pragma warning restore CA2000 // MeterProvider owns exporter lifecycle + + var collectCount = 0; + var firstCollectStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allowFirstCollectToComplete = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var originalCollect = exporter!.Collect; + exporter.Collect = (timeout) => + { + var currentCollectCount = Interlocked.Increment(ref collectCount); + + if (currentCollectCount == 1) + { + firstCollectStarted.SetResult(true); + Assert.True(allowFirstCollectToComplete.Task.Wait(TimeSpan.FromSeconds(5)), "First collection did not resume."); + } + + return originalCollect!(timeout); + }; + + var counter = meter.CreateCounter("test_counter", description: "Help text."); + const int SeriesCount = 4_000; + for (var i = 0; i < SeriesCount; i++) + { + counter.Add( + 1234567890123456789L, + new KeyValuePair("index", $"series-value-{i:D8}-padding-padding-padding")); + } + + var protocol = GetProtocol(openMetricsRequested: false); + + async Task CollectAsync() + { + var response = await EnterCollectAsync(exporter, protocol); + try + { + return response; + } + finally + { + exporter.CollectionManager.ExitCollect(protocol); + } + } + + var firstCollectTask = Task.Run(CollectAsync); + + await firstCollectStarted.Task; + + var secondCollectTask = CollectAsync(); + + Assert.False(secondCollectTask.IsCompleted, "Second collection did not join the active collection."); + + allowFirstCollectToComplete.SetResult(true); + + var timeout = TimeSpan.FromSeconds(5); + var all = Task.WhenAll(firstCollectTask, secondCollectTask); + + using (var cts = new CancellationTokenSource(timeout)) + { + var completion = await Task.WhenAny(all, Task.Delay(timeout, cts.Token)); + Assert.Same(all, completion); + } + + var responses = await all; + + Assert.Equal(1, collectCount); + Assert.All(responses, response => + { + Assert.False(response.Succeeded, "Expected the collection to fail."); + Assert.Equal(0, response.View.Count); + }); + } + [Fact] public async Task OpenMetricsDoesNotEmitScopeInfoMetricFamily() { @@ -877,6 +970,70 @@ public async Task LargeScrapeExpandsBufferAndReturnsCompleteResponse(bool openMe } } + [Theory] + [InlineData(85_000, false)] // The buffer cannot grow beyond its initial size, so the large scrape is dropped. + [InlineData(64 * 1024 * 1024, true)] // Ample budget, so the large scrape is served in full. + public async Task MaxScrapeResponseSizeBytesBoundsTheResponseBuffer(int maxScrapeResponseSizeBytes, bool expectNonEmpty) + { + using var meter = CreateMeter(); + + using var provider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) +#if PROMETHEUS_HTTP_LISTENER + .AddPrometheusHttpListener(options => + { + options.MaxScrapeResponseSizeBytes = maxScrapeResponseSizeBytes; + options.ScrapeResponseCacheDurationMilliseconds = 0; + }) +#elif PROMETHEUS_ASPNETCORE + .AddPrometheusExporter(options => + { + options.MaxScrapeResponseSizeBytes = maxScrapeResponseSizeBytes; + options.ScrapeResponseCacheDurationMilliseconds = 0; + }) +#endif + .Build(); + +#pragma warning disable CA2000 // MeterProvider owns exporter lifecycle + Assert.True(provider.TryFindExporter(out PrometheusExporter? exporter)); +#pragma warning restore CA2000 // MeterProvider owns exporter lifecycle + + // Emit enough unique time series that the serialized output is far larger than the + // 85,000-byte initial scrape buffer, so serving the scrape requires the buffer to grow. + var counter = meter.CreateCounter("test_counter", description: "Help text."); + const int SeriesCount = 4_000; + for (var i = 0; i < SeriesCount; i++) + { + counter.Add( + 1234567890123456789L, + new KeyValuePair("index", $"series-value-{i:D8}-padding-padding-padding")); + } + + var protocol = GetProtocol(openMetricsRequested: false); + var response = await exporter!.CollectionManager.EnterCollect(protocol); + + try + { + if (expectNonEmpty) + { + // The configured budget is large enough for the buffer to grow and serve the scrape. + Assert.True(response.Succeeded, "Expected the collection to succeed."); + Assert.True(response.View.Count > 85_000, $"Expected a complete response, but only {response.View.Count} bytes were written."); + } + else + { + // The buffer cannot grow past the configured maximum, so the scrape is dropped and a + // failed (empty) response is returned rather than allocating without bound. + Assert.False(response.Succeeded, "Expected the collection to fail."); + Assert.Equal(0, response.View.Count); + } + } + finally + { + exporter.CollectionManager.ExitCollect(protocol); + } + } + private static int CountOccurrences(string value, string substring) { var count = 0; @@ -907,7 +1064,9 @@ private static int CountOccurrences(string value, string substring) private static Meter CreateMeter([CallerMemberName] string name = "") => new(name); #if PROMETHEUS_HTTP_LISTENER - private static MeterProvider CreateMeterProviderWithRandomPort(Meter meter) + private static MeterProvider CreateMeterProviderWithRandomPort( + Meter meter, + Action? configure = null) { var retryAttempts = 5; @@ -923,6 +1082,7 @@ private static MeterProvider CreateMeterProviderWithRandomPort(Meter meter) { options.Port = port; options.ScrapeResponseCacheDurationMilliseconds = 0; + configure?.Invoke(options); }) .Build(); } diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs index ffb09eda3ad..4f0b00d05ce 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs @@ -512,6 +512,35 @@ public async Task WhenRequestDeadlineInvalid_Returns200(string scrapeTimeoutSeco Assert.Equal(HttpStatusCode.OK, response.StatusCode); } + [Fact] + public async Task WhenResponseExceedsMaxScrapeResponseSize_Returns500() + { + using var meter = new Meter(MeterName, MeterVersion); + + using var context = CreateMeterProvider( + meter, + configureListener: options => + { + options.Port = GetRandomPort(); + options.MaxScrapeResponseSizeBytes = PrometheusExporterOptions.InitialScrapeResponseSizeBytes; + return options.Port; + }); + + // Emit enough series that the serialized response far exceeds the configured maximum, so + // the response buffer cannot grow to hold it and the scrape fails rather than returning a + // misleading empty 200 response. + for (var x = 0; x < 2_000; x++) + { + meter.CreateCounter("counter_double_" + x, unit: "By").Add(1); + } + + using var client = new HttpClient { BaseAddress = context.BaseAddress }; + + using var response = await client.GetAsync(new Uri("metrics", UriKind.Relative)); + + Assert.Equal(HttpStatusCode.InternalServerError, response.StatusCode); + } + internal static MeterProviderTestContext CreateMeterProvider( Meter meter, Func? configureListener = null,