Skip to content

Reduce dashboard CPU under sustained metric ingestion #20736

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
James Newton-King (JamesNK) merged 1 commit into from
Oct 5, 2026
Merged

Reduce dashboard CPU under sustained metric ingestion #20736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 83 additions & 23 deletions benchmarks/Aspire.Dashboard.Benchmarks/TelemetryRepositoryMetricsBenchmarks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
using Google.Protobuf;
using Google.Protobuf.Collections;
using Microsoft.Extensions.Logging.Abstractions;
Expand All @@ -23,14 +24,14 @@
namespace Aspire.Dashboard.Benchmarks;

[MemoryDiagnoser]
[ThreadingDiagnoser]
[Config(typeof(Config))]
public class TelemetryRepositoryMetricsBenchmarks
{
private const int MetricSamplesPerBatch = 100;
private const string MetricMeterName = "benchmark-meter";
private const string MetricInstrumentName = "benchmark.metric";
private const string HistogramMetricInstrumentName = "benchmark.histogram";
private static readonly double[] s_histogramObservations = [5, 25, 75, 150];
private static readonly TimeSpan s_metricDataDuration = TimeSpan.FromHours(6);
private static readonly TimeSpan s_metricDisplayDuration = TimeSpan.FromHours(6);
private static readonly TimeSpan s_metricInterval = TimeSpan.FromSeconds(2);
Expand All @@ -43,6 +44,8 @@ public class TelemetryRepositoryMetricsBenchmarks
private DashboardSqliteDatabase _database = null!;
private SqliteTelemetryRepository _queryRepository = null!;
private IReadOnlyList<MetricDimensionCursor> _incrementalCursors = null!;
private RepeatedField<ResourceMetrics> _ingestionMetrics = null!;
private HistogramDataPoint[] _ingestionPoints = null!;

[Params(1, 5)]
public int DimensionCount { get; set; }
Expand All @@ -65,13 +68,50 @@ public class TelemetryRepositoryMetricsBenchmarks
[GlobalSetup(Target = nameof(GetHistogramMetricsIncrementalRollup))]
public Task SetupHistogramMetricsIncrementalRollup() => SetupIncrementalAsync(isHistogram: true, HistogramMetricInstrumentName);

private async Task SetupAsync(bool isHistogram)
[GlobalSetup(Target = nameof(AddHistogramMetricsAtCapacity))]
public async Task SetupHistogramMetricsIngestion()
{
await InitializeRepositoryAsync();
var retainedPointCount = new TelemetryLimitOptions().MaxMetricsCount;
var startTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var exemplars = s_histogramObservations.Select(value => CreateMetricExemplar(startTime, value)).ToArray();
var context = new AddContext();
foreach (var batch in CreateLongDurationMetricBatches(DimensionCount, isHistogram: true, retainedPointCount))
{
foreach (var point in batch[0].ScopeMetrics[0].Metrics[0].Histogram.DataPoints)
{
// Cumulative reservoirs replay unrefreshed exemplars. Include them in every export to
// exercise point eviction and its cascading exemplar deletes in a long-running app.
point.Exemplars.Clear();
point.Exemplars.Add(exemplars);
}
await _queryRepository.AddMetricsAsync(context, batch);
_ingestionMetrics = batch;
}
if (context.FailureCount > 0)
{
throw new InvalidOperationException($"Failed to add {context.FailureCount} benchmark metric points.");
}

var points = _ingestionMetrics[0].ScopeMetrics[0].Metrics[0].Histogram.DataPoints;
_ingestionPoints = points.TakeLast(DimensionCount).ToArray();
points.Clear();
points.Add(_ingestionPoints);
}

private async Task InitializeRepositoryAsync()
{
_temporaryDirectory = Directory.CreateTempSubdirectory("aspire-dashboard-metrics-benchmark-").FullName;
_database = new DashboardSqliteDatabase(Path.Combine(_temporaryDirectory, "query.db"));
await _database.InitializeSchemaAsync(CancellationToken.None);
_queryRepository = CreateRepository(_database);
}

private async Task SetupAsync(bool isHistogram)
{
await InitializeRepositoryAsync();
var addContext = new AddContext();
foreach (var batch in CreateLongDurationMetricBatches(DimensionCount, isHistogram))
foreach (var batch in CreateLongDurationMetricBatches(DimensionCount, isHistogram, (int)(s_metricDataDuration / s_metricInterval)))
{
await _queryRepository.AddMetricsAsync(addContext, batch);
}
Expand All @@ -84,7 +124,7 @@ private async Task SetupAsync(bool isHistogram)
private async Task SetupIncrementalAsync(bool isHistogram, string instrumentName)
{
await SetupAsync(isHistogram);
var instrument = GetLongDurationInstrument(instrumentName, s_metricDataPointInterval);
var instrument = await GetLongDurationInstrumentAsync(instrumentName, s_metricDataPointInterval);
_incrementalCursors = instrument.Dimensions.Select(dimension =>
{
var latestValue = dimension.Values[^1];
Expand All @@ -106,57 +146,78 @@ public void Cleanup()
}

[Benchmark(Description = "TelemetryRepository: query 6h metrics display")]
public int GetMetricsLongDuration()
public async Task<int> GetMetricsLongDuration()
{
var instrument = GetLongDurationInstrument(MetricInstrumentName);
var instrument = await GetLongDurationInstrumentAsync(MetricInstrumentName);

return instrument.Dimensions.Sum(dimension => dimension.Values.Count);
}

[Benchmark(Description = "TelemetryRepository: query 6h histogram metrics display")]
public int GetHistogramMetricsLongDuration()
public async Task<int> GetHistogramMetricsLongDuration()
{
var instrument = GetLongDurationInstrument(HistogramMetricInstrumentName);
var instrument = await GetLongDurationInstrumentAsync(HistogramMetricInstrumentName);

return instrument.Dimensions.Sum(dimension =>
dimension.Values.Count + dimension.Values.Sum(value => value.Exemplars.Count));
}

[Benchmark(Description = "TelemetryRepository: query 6h metrics with dashboard rollup")]
public int GetMetricsLongDurationRollup()
public async Task<int> GetMetricsLongDurationRollup()
{
var instrument = GetLongDurationInstrument(MetricInstrumentName, s_metricDataPointInterval);
var instrument = await GetLongDurationInstrumentAsync(MetricInstrumentName, s_metricDataPointInterval);

return instrument.Dimensions.Sum(dimension => dimension.Values.Count);
}

[Benchmark(Description = "TelemetryRepository: query 6h histogram metrics with dashboard rollup")]
public int GetHistogramMetricsLongDurationRollup()
public async Task<int> GetHistogramMetricsLongDurationRollup()
{
var instrument = GetLongDurationInstrument(HistogramMetricInstrumentName, s_metricDataPointInterval);
var instrument = await GetLongDurationInstrumentAsync(HistogramMetricInstrumentName, s_metricDataPointInterval);

return instrument.Dimensions.Sum(dimension =>
dimension.Values.Count + dimension.Values.Sum(value => value.Exemplars.Count));
}

[Benchmark(Description = "TelemetryRepository: query incremental metrics with dashboard rollup")]
public int GetMetricsIncrementalRollup()
public async Task<int> GetMetricsIncrementalRollup()
{
var instrument = GetLongDurationInstrument(MetricInstrumentName, s_metricDataPointInterval, _incrementalCursors);
var instrument = await GetLongDurationInstrumentAsync(MetricInstrumentName, s_metricDataPointInterval, _incrementalCursors);

return instrument.Dimensions.Sum(dimension => dimension.Values.Count);
}

[Benchmark(Description = "TelemetryRepository: query incremental histogram metrics with dashboard rollup")]
public int GetHistogramMetricsIncrementalRollup()
public async Task<int> GetHistogramMetricsIncrementalRollup()
{
var instrument = GetLongDurationInstrument(HistogramMetricInstrumentName, s_metricDataPointInterval, _incrementalCursors);
var instrument = await GetLongDurationInstrumentAsync(HistogramMetricInstrumentName, s_metricDataPointInterval, _incrementalCursors);

return instrument.Dimensions.Sum(dimension =>
dimension.Values.Count + dimension.Values.Sum(value => value.Exemplars.Count));
}

private OtlpInstrumentData GetLongDurationInstrument(
[Benchmark(Description = "TelemetryRepository: ingest histogram metrics at retention capacity")]
public async Task<int> AddHistogramMetricsAtCapacity()
{
foreach (var point in _ingestionPoints)
{
point.Count++;
point.Sum += s_histogramObservations[0];
point.BucketCounts[0]++;
point.TimeUnixNano += (ulong)s_metricInterval.Ticks * 100;
}

var context = new AddContext();
await _queryRepository.AddMetricsAsync(context, _ingestionMetrics);
if (context.SuccessCount != DimensionCount || context.FailureCount > 0)
{
throw new InvalidOperationException($"Expected {DimensionCount} benchmark metric points, added {context.SuccessCount} and rejected {context.FailureCount}.");
}

return context.SuccessCount;
}

private async Task<OtlpInstrumentData> GetLongDurationInstrumentAsync(
string instrumentName,
TimeSpan? dataPointInterval = null,
IReadOnlyList<MetricDimensionCursor>? dimensionCursors = null)
Expand All @@ -165,7 +226,7 @@ private OtlpInstrumentData GetLongDurationInstrument(
?? throw new InvalidOperationException($"Unable to find the benchmark metric '{instrumentName}' end time.");

// Match the dashboard metrics display query, which includes one preceding rollup for histogram calculations.
return _queryRepository.GetInstrument(new GetInstrumentRequest
return await _queryRepository.GetInstrumentAsync(new GetInstrumentRequest
{
ResourceKey = s_metricResourceKey,
MeterName = MetricMeterName,
Expand All @@ -175,7 +236,7 @@ private OtlpInstrumentData GetLongDurationInstrument(
DataPointInterval = dataPointInterval,
PopulateExemplarAttributes = false,
DimensionCursors = dimensionCursors ?? []
}) ?? throw new InvalidOperationException($"Unable to find the benchmark metric '{instrumentName}'.");
}, cancellationToken: CancellationToken.None) ?? throw new InvalidOperationException($"Unable to find the benchmark metric '{instrumentName}'.");
}

private static SqliteTelemetryRepository CreateRepository(DashboardSqliteDatabase database)
Expand All @@ -189,13 +250,12 @@ private static SqliteTelemetryRepository CreateRepository(DashboardSqliteDatabas
[]);
}

private static IEnumerable<RepeatedField<ResourceMetrics>> CreateLongDurationMetricBatches(int dimensionCount, bool isHistogram)
private static IEnumerable<RepeatedField<ResourceMetrics>> CreateLongDurationMetricBatches(int dimensionCount, bool isHistogram, int totalSampleCount)
{
var startTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
double[] observations = [5, 25, 75, 150];
var observations = s_histogramObservations;
var bucketCounts = Enumerable.Range(0, dimensionCount).Select(_ => new ulong[observations.Length]).ToArray();
var sums = new double[dimensionCount];
var totalSampleCount = (int)(s_metricDataDuration / s_metricInterval);

for (var firstSampleIndex = 0; firstSampleIndex < totalSampleCount; firstSampleIndex += MetricSamplesPerBatch)
{
Expand Down Expand Up @@ -386,7 +446,7 @@ private sealed class Config : ManualConfig
{
public Config()
{
AddJob(Job.Dry);
AddJob(Job.Dry.WithToolchain(InProcessNoEmitToolchain.Instance).DontEnforcePowerPlan());

AddDiagnoser(MemoryDiagnoser.Default);
}
Expand Down
2 changes: 2 additions & 0 deletions docs/specs/dashboard-persistence.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,8 @@ The Dashboard applies these default ingestion limits:

The oldest logs, traces, and metric points are removed when their limits are exceeded. Fixed limits of 10,000 also apply to resource views per resource, scopes per database, instruments per resource, and dimensions per instrument; additional identities are rejected.

The `AddHistogramMetricsAtCapacity` benchmark in [TelemetryRepositoryMetricsBenchmarks](../../benchmarks/Aspire.Dashboard.Benchmarks/TelemetryRepositoryMetricsBenchmarks.cs) measures steady-state ingestion with one or five dimensions already at the default retention limit. Each invocation adds one cumulative histogram point per dimension with four replayed exemplars, exercising eviction and cascading exemplar deletes. Payload construction and history population are outside the measured operation.

Console logs are persisted only after their stream is viewed or exported. The frontend keeps up to `Dashboard:Frontend:MaxConsoleLogCount` entries in memory, which defaults to 100,000. The same limit applies across all console logs in the database, with the oldest entries removed when the limit is exceeded. Historical runs can therefore omit uncaptured logs.

### Database file size
Expand Down
30 changes: 20 additions & 10 deletions src/Aspire.Dashboard/Otlp/Storage/SqliteTelemetryRepository.Metrics.Writes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,7 @@ FROM updates
foreach (var point in pointBatch.Inserts)
{
point.Context.SuccessCount += point.SourcePointCount;
point.Dimension.PointCount++;

if (ReferenceEquals(point.Dimension.PendingPoint, point))
{
Expand Down Expand Up @@ -423,6 +424,7 @@ private MetricDimensionState GetOrAddMetricDimension(
var dimensions = connection.Query<MetricDimensionStateRecord>("""
SELECT
d.dimension_id AS DimensionId,
(SELECT COUNT(*) FROM telemetry_metric_points WHERE dimension_id = d.dimension_id) AS PointCount,
a.attribute_key AS AttributeKey,
a.attribute_value AS AttributeValue,
p.point_id AS PointId,
Expand Down Expand Up @@ -451,6 +453,7 @@ LIMIT 1
return new MetricDimensionState
{
DimensionId = group.Key,
PointCount = first.PointCount,
Attributes = group
.Where(record => record.AttributeKey is not null)
.Select(record => KeyValuePair.Create(record.AttributeKey!, record.AttributeValue!))
Expand Down Expand Up @@ -733,22 +736,27 @@ INSERT OR IGNORE INTO telemetry_metric_exemplars (

private void TrimMetricDimensions(SqliteConnection connection, IDbTransaction transaction, IEnumerable<MetricDimensionState> dimensions)
{
foreach (var batch in dimensions.Chunk(MaxMetricPointBatchSize))
foreach (var dimension in dimensions)
{
var removeCount = dimension.PointCount - _otlpContext.Options.MaxMetricsCount;
if (removeCount <= 0)
{
continue;
}

// Count cached points instead of ranking the entire history on every insert. The dimension-order
// index lets SQLite visit only the oldest surplus points, even after the retention limit is reached.
connection.Execute("""
DELETE FROM telemetry_metric_points
WHERE point_id IN (
SELECT point_id
FROM (
SELECT
point_id,
ROW_NUMBER() OVER (PARTITION BY dimension_id ORDER BY point_id DESC) AS point_rank
FROM telemetry_metric_points
WHERE dimension_id IN @DimensionIds
)
WHERE point_rank > @MaxMetricsCount
FROM telemetry_metric_points
WHERE dimension_id = @DimensionId
ORDER BY point_id
LIMIT @RemoveCount
);
""", new { DimensionIds = batch.Select(dimension => dimension.DimensionId).ToArray(), _otlpContext.Options.MaxMetricsCount }, transaction);
""", new { dimension.DimensionId, RemoveCount = removeCount }, transaction);
dimension.PointCount -= removeCount;
}
}

Expand Down Expand Up @@ -861,6 +869,7 @@ private sealed record PendingMetricDimensionAttribute(MetricDimensionState Dimen
private sealed class MetricDimensionState
{
public long DimensionId { get; set; }
public long PointCount { get; set; }
public required KeyValuePair<string, string>[] Attributes { get; init; }
public MetricPointRecord? LatestPoint { get; set; }
public PendingMetricPoint? PendingPoint { get; set; }
Expand Down Expand Up @@ -940,6 +949,7 @@ private sealed class PendingMetricPoint
internal sealed class MetricDimensionStateRecord
{
public required long DimensionId { get; init; }
public required long PointCount { get; init; }
public string? AttributeKey { get; init; }
public string? AttributeValue { get; init; }
public long? PointId { get; init; }
Expand Down
Loading
Loading