Skip to content

[release/13.3] Fix unbounded collection growth in TelemetryRepository #16590

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jose Perez Rodriguez (joperezr) merged 5 commits into from
May 1, 2026
Merged

[release/13.3] Fix unbounded collection growth in TelemetryRepository #16590

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6562029
Fix unbounded collection growth in TelemetryRepository and related types
JamesNK Apr 30, 2026
0f41aed
Add limit enforcement tests and fix uncaught exceptions from resource…
JamesNK Apr 30, 2026
8b8f83b
Reset HasTraces/HasLogs and clear unviewed error logs on full clear
JamesNK Apr 30, 2026
93863e4
Remove redundant per-telemetry-type resource limit tests
JamesNK Apr 30, 2026
f328daf
Add scope and instrument limits with TryGetValue pattern
JamesNK May 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/Aspire.Dashboard/Configuration/DashboardOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ public sealed class TelemetryLimitOptions
public int MaxAttributeCount { get; set; } = 128;
public int MaxAttributeLength { get; set; } = int.MaxValue;
public int MaxSpanEventCount { get; set; } = int.MaxValue;
public int MaxResourceCount { get; set; } = 10_000;
}

public sealed class UIOptions
Expand Down
22 changes: 13 additions & 9 deletions src/Aspire.Dashboard/Otlp/Model/OtlpHelpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
Expand Down Expand Up @@ -467,18 +466,23 @@ public static bool TryGetOrAddScope(Dictionary<string, OtlpScope> scopes, Instru
// Semantically when InstrumentationScope isn't set, it is equivalent with
// an empty instrumentation scope name (unknown).
var name = scope?.Name ?? string.Empty;
ref var scopeRef = ref CollectionsMarshal.GetValueRefOrAddDefault(scopes, name, out _);
// Adds to dictionary if not present.
if (scopeRef == null)
if (scopes.TryGetValue(name, out s))
{
scopeRef = (scope != null)
? new OtlpScope(scope.Name, scope.Version, scope.Attributes.ToKeyValuePairs(context))
: OtlpScope.Empty;
return true;
}

context.Logger.LogTrace("Added scope '{ScopeName}' to {TelemetryType}.", scopeRef.Name, telemetryType);
if (scopes.Count >= TelemetryRepository.MaxScopeCount)
{
throw new InvalidOperationException($"Scope limit of {TelemetryRepository.MaxScopeCount} reached for {telemetryType}. Scope '{name}' will not be added.");
}

s = scopeRef;
s = (scope != null)
? new OtlpScope(scope.Name, scope.Version, scope.Attributes.ToKeyValuePairs(context))
: OtlpScope.Empty;

scopes.Add(name, s);

context.Logger.LogTrace("Added scope '{ScopeName}' to {TelemetryType}.", s.Name, telemetryType);
return true;
}
catch (Exception ex)
Expand Down
23 changes: 18 additions & 5 deletions src/Aspire.Dashboard/Otlp/Model/OtlpInstrument.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Diagnostics.CodeAnalysis;
using System.Runtime.InteropServices;
using Aspire.Dashboard.Otlp.Model.MetricValues;
using Aspire.Dashboard.Otlp.Storage;
using Google.Protobuf.Collections;
using OpenTelemetry.Proto.Common.V1;

Expand Down Expand Up @@ -63,6 +64,11 @@ public DimensionScope FindScope(RepeatedField<KeyValue> attributes, ref KeyValue
// Need to add dimensions using durable attributes instance after scope is created.
if (!Dimensions.TryGetValue(comparableAttributes, out var dimension))
{
if (Dimensions.Count >= TelemetryRepository.MaxDimensionCount)
{
throw new InvalidOperationException($"Dimension limit of {TelemetryRepository.MaxDimensionCount} reached for instrument '{Summary.Name}'.");
}

dimension = CreateDimensionScope(comparableAttributes);
Dimensions.Add(dimension.Attributes, dimension);
}
Expand All @@ -78,28 +84,35 @@ private DimensionScope CreateDimensionScope(Memory<KeyValuePair<string, string>>
var keys = KnownAttributeValues.Keys.Union(durableAttributes.Select(a => a.Key)).Distinct();
foreach (var key in keys)
{
ref var values = ref CollectionsMarshal.GetValueRefOrAddDefault(KnownAttributeValues, key, out _);
ref var values = ref CollectionsMarshal.GetValueRefOrAddDefault(KnownAttributeValues, key, out var existed);
// Adds to dictionary if not present.
if (values == null)
{
if (!existed && KnownAttributeValues.Count > TelemetryRepository.MaxKnownAttributeValueCount)
{
// Over limit. Remove the default entry that GetValueRefOrAddDefault added.
KnownAttributeValues.Remove(key);
continue;
}

values = new List<string?>();

// If the key is new and there are already dimensions, add an empty value because there are dimensions without this key.
if (!isFirst)
{
TryAddValue(values, null);
TryAddValue(values, null, TelemetryRepository.MaxKnownAttributeValuesPerKey);
}
}

var currentDimensionValue = OtlpHelpers.GetValue(durableAttributes, key);
TryAddValue(values, currentDimensionValue);
TryAddValue(values, currentDimensionValue, TelemetryRepository.MaxKnownAttributeValuesPerKey);
}

return dimension;

static void TryAddValue(List<string?> values, string? value)
static void TryAddValue(List<string?> values, string? value, int maxValues)
{
if (!values.Contains(value))
if (values.Count < maxValues && !values.Contains(value))
{
values.Add(value);
}
Expand Down
33 changes: 23 additions & 10 deletions src/Aspire.Dashboard/Otlp/Model/OtlpResource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.InteropServices;
using Aspire.Dashboard.Otlp.Storage;
using Google.Protobuf.Collections;
using OpenTelemetry.Proto.Common.V1;
Expand Down Expand Up @@ -45,6 +44,7 @@ public class OtlpResource : IOtlpResource
public ResourceKey ResourceKey => new ResourceKey(ResourceName, InstanceId);

private readonly ReaderWriterLockSlim _metricsLock = new();
// Bounded by TelemetryRepository.MaxScopeCount. Cleared when metrics are cleared.
private readonly Dictionary<string, OtlpScope> _meters = new();
private readonly Dictionary<OtlpInstrumentKey, OtlpInstrument> _instruments = new();
private readonly ConcurrentDictionary<KeyValuePair<string, string>[], OtlpResourceView> _resourceViews = new(ResourceViewKeyComparer.Instance);
Expand All @@ -70,7 +70,7 @@ public void AddMetrics(AddContext context, RepeatedField<ScopeMetrics> scopeMetr
{
if (!OtlpHelpers.TryGetOrAddScope(_meters, sm.Scope, Context, TelemetryType.Metrics, out var scope))
{
context.FailureCount += sm.Metrics.Count;
context.FailureCount += sm.Metrics.Sum(m => GetMetricDataPointCount(m));
continue;
}

Expand All @@ -86,11 +86,13 @@ public void AddMetrics(AddContext context, RepeatedField<ScopeMetrics> scopeMetr
}

var instrumentKey = new OtlpInstrumentKey(scope.Name, metric.Name);
ref var instrumentRef = ref CollectionsMarshal.GetValueRefOrAddDefault(_instruments, instrumentKey, out _);
if (instrumentRef == null)
if (_instruments.TryGetValue(instrumentKey, out var existingInstrument))
{
// Adds to dictionary if not present.
instrumentRef = new OtlpInstrument
instrument = existingInstrument;
}
else if (_instruments.Count < TelemetryRepository.MaxInstrumentCount)
{
var newInstrument = new OtlpInstrument
{
Summary = new OtlpInstrumentSummary
{
Expand All @@ -104,10 +106,15 @@ public void AddMetrics(AddContext context, RepeatedField<ScopeMetrics> scopeMetr
Context = Context
};

Context.Logger.LogTrace("Added metric instrument '{InstrumentName}' for scope '{ScopeName}'.", instrumentRef.Summary.Name, scope.Name);
}
_instruments.Add(instrumentKey, newInstrument);
instrument = newInstrument;

instrument = instrumentRef;
Context.Logger.LogTrace("Added metric instrument '{InstrumentName}' for scope '{ScopeName}'.", instrument.Summary.Name, scope.Name);
}
else
{
throw new InvalidOperationException($"Instrument limit of {TelemetryRepository.MaxInstrumentCount} reached. Instrument '{metric.Name}' will not be added.");
}
}
catch (Exception ex)
{
Expand All @@ -127,7 +134,7 @@ public void AddMetrics(AddContext context, RepeatedField<ScopeMetrics> scopeMetr
}
}

private static int GetMetricDataPointCount(Metric metric)
internal static int GetMetricDataPointCount(Metric metric)
{
return metric.DataCase switch
{
Expand Down Expand Up @@ -207,6 +214,7 @@ public void ClearMetrics()
try
{
_instruments.Clear();
_meters.Clear();
}
finally
{
Expand Down Expand Up @@ -296,6 +304,11 @@ internal OtlpResourceView GetView(RepeatedField<KeyValue> attributes)
return resourceView;
}

if (_resourceViews.Count >= TelemetryRepository.MaxResourceViewCount)
{
throw new InvalidOperationException($"Resource view limit of {TelemetryRepository.MaxResourceViewCount} reached.");
}

return _resourceViews.GetOrAdd(view.Properties, view);
}

Expand Down
96 changes: 84 additions & 12 deletions src/Aspire.Dashboard/Otlp/Storage/TelemetryRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ namespace Aspire.Dashboard.Otlp.Storage;

public sealed partial class TelemetryRepository : IDisposable
{
internal const int MaxResourceViewCount = 10_000;
internal const int MaxInstrumentCount = 10_000;
internal const int MaxScopeCount = 10_000;
internal const int MaxDimensionCount = 10_000;
internal const int MaxKnownAttributeValueCount = 10_000;
internal const int MaxKnownAttributeValuesPerKey = 10_000;

private readonly PauseManager _pauseManager;
private readonly IOutgoingPeerResolver[] _outgoingPeerResolvers;
private readonly ILogger _logger;
Expand All @@ -46,15 +53,21 @@ public sealed partial class TelemetryRepository : IDisposable
private readonly ConcurrentDictionary<ResourceKey, OtlpResource> _resources = new();

private readonly ReaderWriterLockSlim _logsLock = new();
// Bounded by MaxScopeCount. Cleared when all logs are cleared.
private readonly Dictionary<string, OtlpScope> _logScopes = new();
private readonly CircularBuffer<OtlpLogEntry> _logs;
// Bounded by _resources count * MaxAttributeCount. Cleared per-resource or when all logs are cleared.
private readonly HashSet<(OtlpResource Resource, string PropertyKey)> _logPropertyKeys = new();
// Bounded by _resources count * MaxAttributeCount. Cleared per-resource or when all traces are cleared.
private readonly HashSet<(OtlpResource Resource, string PropertyKey)> _tracePropertyKeys = new();
private readonly Dictionary<ResourceKey, int> _resourceUnviewedErrorLogs = new();

private readonly ReaderWriterLockSlim _tracesLock = new();
// Bounded by MaxScopeCount. Cleared when all traces are cleared.
private readonly Dictionary<string, OtlpScope> _traceScopes = new();
private readonly CircularBuffer<OtlpTrace> _traces;
// Not explicitly capped per add — bounded only by the sum of span links across in-buffer traces.
// Cleaned up on trace eviction and clear, so growth is limited by the circular buffer capacity.
private readonly List<OtlpSpanLink> _spanLinks = new();
private readonly List<IDisposable> _peerResolverSubscriptions = new();
internal readonly OtlpContext _otlpContext;
Expand Down Expand Up @@ -237,6 +250,14 @@ private OtlpResourceView GetOrAddResourceView(Resource resource)
return (Resource: resource, IsNew: false);
}

// Check resource limit before adding a new resource.
// Note: This is a soft cap. Concurrent callers may both pass this check and slightly exceed the limit
// because _resources is a ConcurrentDictionary and the count check + GetOrAdd are not atomic.
if (_resources.Count >= _otlpContext.Options.MaxResourceCount)
{
throw new InvalidOperationException($"Resource limit of {_otlpContext.Options.MaxResourceCount} reached. Resource '{key}' will not be added.");
Comment thread
JamesNK marked this conversation as resolved.
}

// Slower get or add path.
// This GetOrAdd allocates a closure, so we avoid it if possible.
var newResource = false;
Expand Down Expand Up @@ -323,7 +344,7 @@ public void AddLogs(AddContext context, RepeatedField<ResourceLogs> resourceLogs
}
catch (Exception ex)
{
context.FailureCount += rl.ScopeLogs.Count;
context.FailureCount += rl.ScopeLogs.Sum(s => s.LogRecords.Count);
_otlpContext.Logger.LogInformation(ex, "Error adding resource.");
continue;
}
Expand Down Expand Up @@ -792,22 +813,41 @@ public void ClearTraces(ResourceKey? resourceKey = null)
{
// Nothing selected, clear everything.
_traces.Clear();
_traceScopes.Clear();
_tracePropertyKeys.Clear();
_spanLinks.Clear();

foreach (var resource in _resources.Values)
{
SetResourceHasTraces(resource, false);
}
}
Comment thread
JamesNK marked this conversation as resolved.
else
{
for (var i = _traces.Count - 1; i >= 0; i--)
{
var trace = _traces[i];
// Remove trace if any span matches one of the resources. This matches filter behavior.
if (MatchResources(_traces[i], resources))
if (MatchResources(trace, resources))
{
// Remove span links for the removed trace.
foreach (var span in trace.Spans)
{
foreach (var link in span.Links)
{
_spanLinks.Remove(link);
Comment thread
JamesNK marked this conversation as resolved.
}
}
Comment thread
JamesNK marked this conversation as resolved.

_traces.RemoveAt(i);
continue;
}
}

// Update HasTraces flag for cleared resources
// Remove property keys for cleared resources.
foreach (var resource in resources)
{
_tracePropertyKeys.RemoveWhere(k => k.Resource.ResourceKey == resource.ResourceKey);
SetResourceHasTraces(resource, false);
}
}
Expand Down Expand Up @@ -836,6 +876,15 @@ public void ClearStructuredLogs(ResourceKey? resourceKey = null)
{
// Nothing selected, clear everything.
_logs.Clear();
_logScopes.Clear();
_logPropertyKeys.Clear();
Comment thread
JamesNK marked this conversation as resolved.

foreach (var resource in _resources.Values)
{
SetResourceHasLogs(resource, false);
}

_resourceUnviewedErrorLogs.Clear();
}
else
{
Expand All @@ -848,9 +897,10 @@ public void ClearStructuredLogs(ResourceKey? resourceKey = null)
}
}

// Update HasLogs flag for cleared resources
// Update HasLogs flag and remove property keys for cleared resources.
foreach (var resource in resources)
{
_logPropertyKeys.RemoveWhere(k => k.Resource.ResourceKey == resource.ResourceKey);
SetResourceHasLogs(resource, false);
_resourceUnviewedErrorLogs.Remove(resource.ResourceKey);
}
Expand Down Expand Up @@ -1066,7 +1116,7 @@ public void AddMetrics(AddContext context, RepeatedField<ResourceMetrics> resour
}
catch (Exception ex)
{
context.FailureCount += rm.ScopeMetrics.Sum(s => s.Metrics.Count);
context.FailureCount += rm.ScopeMetrics.Sum(sm => sm.Metrics.Sum(OtlpResource.GetMetricDataPointCount));
_otlpContext.Logger.LogInformation(ex, "Error adding resource.");
continue;
}
Expand Down Expand Up @@ -1316,9 +1366,17 @@ static bool TryGetTraceById(CircularBuffer<OtlpTrace> traces, ReadOnlyMemory<byt
return null;
}

var resourceKey = ResourceKey.Create(name: peer.DisplayName, instanceId: peer.Name);
var (resource, _) = GetOrAddResource(resourceKey, uninstrumentedPeer: true);
return resource;
try
{
var resourceKey = ResourceKey.Create(name: peer.DisplayName, instanceId: peer.Name);
var (resource, _) = GetOrAddResource(resourceKey, uninstrumentedPeer: true);
return resource;
}
catch (Exception ex)
{
_logger.LogInformation(ex, "Error adding peer resource.");
return null;
}
}

private void CalculateTraceUninstrumentedPeers(OtlpTrace trace)
Expand All @@ -1338,9 +1396,16 @@ private void CalculateTraceUninstrumentedPeers(OtlpTrace trace)
continue;
}

var resourceKey = ResourceKey.Create(name: uninstrumentedPeer.DisplayName, instanceId: uninstrumentedPeer.Name);
var (resource, _) = GetOrAddResource(resourceKey, uninstrumentedPeer: true);
trace.SetSpanUninstrumentedPeer(span, resource);
try
{
var resourceKey = ResourceKey.Create(name: uninstrumentedPeer.DisplayName, instanceId: uninstrumentedPeer.Name);
var (resource, _) = GetOrAddResource(resourceKey, uninstrumentedPeer: true);
trace.SetSpanUninstrumentedPeer(span, resource);
}
catch (Exception ex)
{
_logger.LogInformation(ex, "Error adding uninstrumented peer resource.");
Comment thread
JamesNK marked this conversation as resolved.
}
}
else
{
Expand Down Expand Up @@ -1566,7 +1631,14 @@ private Task OnPeerChanged()
// When peers change then we need to recalculate the uninstrumented peers of spans.
foreach (var trace in _traces)
{
CalculateTraceUninstrumentedPeers(trace);
try
{
CalculateTraceUninstrumentedPeers(trace);
}
catch (Exception ex)
{
_logger.LogInformation(ex, "Error recalculating uninstrumented peers.");
}
}
}
finally
Expand Down
Loading
Loading