Add metrics overflow warning to the dashboard (#7784)

Author: JamesNK

playground/Stress/Stress.ApiService/Program.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Diagnostics;
+using System.Globalization;
 using System.Text;
 using System.Text.Json.Nodes;
 using System.Threading.Channels;
@@ -55,6 +56,21 @@
     return "Counter incremented";
 });
 
+app.MapGet("/overflow-counter", (TestMetrics metrics) =>
+{
+    // Emit measurements to ensure at least 2000 unique tag values are emitted,
+    // matching the default cardinality limit in OpenTelemetry.
+    for (var i = 0; i < 250; i++)
+    {
+        for (int j = 0; j < 10; j++)
+        {
+            metrics.IncrementCounter(1, new TagList([new KeyValuePair<string, object?>($"add-tag-{i}", j.ToString(CultureInfo.InvariantCulture))]));
+        }
+    }
+
+    return "Counter overflowed";
+});
+
 app.MapGet("/big-trace", async () =>
 {
     var traceCreator = new TraceCreator

playground/Stress/Stress.ApiService/TestMetrics.cs

@@ -20,7 +20,7 @@ public TestMetrics()
             new KeyValuePair<string, object?>("meter-tag", Guid.NewGuid().ToString())
         ]);
 
-        _counter = _meter.CreateCounter<int>("test-counter", unit: null, description: null, tags:
+        _counter = _meter.CreateCounter<int>("test-counter", unit: null, description: "This is a description", tags:
         [
             new KeyValuePair<string, object?>("instrument-tag", Guid.NewGuid().ToString())
         ]);

playground/Stress/Stress.AppHost/Program.cs

@@ -26,7 +26,10 @@
     }
 }
 
-var serviceBuilder = builder.AddProject<Projects.Stress_ApiService>("stress-apiservice", launchProfileName: null);
+// TODO: OTEL env var can be removed when OTEL libraries are updated to 1.9.0
+// See https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/RELEASENOTES.md#1100
+var serviceBuilder = builder.AddProject<Projects.Stress_ApiService>("stress-apiservice", launchProfileName: null)
+    .WithEnvironment("OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE", "true");
 serviceBuilder.WithCommand(
     name: "icon-test",
     displayName: "Icon test",
@@ -59,6 +62,7 @@
 serviceBuilder.WithHttpCommand("/log-message", "Log message", method: HttpMethod.Get, iconName: "ContentViewGalleryLightning");
 serviceBuilder.WithHttpCommand("/log-message-limit", "Log message limit", method: HttpMethod.Get, iconName: "ContentViewGalleryLightning");
 serviceBuilder.WithHttpCommand("/multiple-traces-linked", "Multiple traces linked", method: HttpMethod.Get, iconName: "ContentViewGalleryLightning");
+serviceBuilder.WithHttpCommand("/overflow-counter", "Overflow counter", method: HttpMethod.Get, iconName: "ContentViewGalleryLightning");
 
 builder.AddProject<Projects.Stress_TelemetryService>("stress-telemetryservice");
 

src/Aspire.Dashboard/Components/Controls/Chart/ChartContainer.razor

@@ -15,6 +15,20 @@ else
         <div class="metrics-chart-header">
             <h3>@_instrument.Summary.Name</h3>
             <p>@_instrument.Summary.Description</p>
+            @if (_instrument.HasOverflow)
+            {
+                <div class="overflow-warning">
+                    <div class="overflow-warning-icon">
+                        <FluentIcon Value="new Icons.Filled.Size16.ErrorCircle()" Color="Color.Error" />
+                    </div>
+
+                    <div class="overflow-warning-message">
+                        <span class="title">@Loc[nameof(ControlsStrings.ChartContainerOverflowTitle)]</span>
+                        @Loc[nameof(ControlsStrings.ChartContainerOverflowDescription)]<br/>
+                        @((MarkupString)string.Format(ControlsStrings.ChartContainerOverflowLink, "https://aka.ms/dotnet/aspire/cardinality-limits"))
+                    </div>
+                </div>
+            }
         </div>
         <FluentTabs ActiveTabId="@($"tab-{ActiveView}")" OnTabChange="@OnTabChangeAsync">
             <FluentTab LabelClass="tab-label"

src/Aspire.Dashboard/Components/Controls/Chart/ChartContainer.razor.cs

@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections.Immutable;
+using System.Runtime.InteropServices;
 using Aspire.Dashboard.Model;
 using Aspire.Dashboard.Otlp.Model;
 using Aspire.Dashboard.Otlp.Model.MetricValues;
@@ -41,7 +43,7 @@ public partial class ChartContainer : ComponentBase, IAsyncDisposable
     [Inject]
     public required ThemeManager ThemeManager { get; init; }
 
-    public List<DimensionFilterViewModel> DimensionFilters { get; } = [];
+    public ImmutableList<DimensionFilterViewModel> DimensionFilters { get; set; } = [];
     public string? PreviousMeterName { get; set; }
     public string? PreviousInstrumentName { get; set; }
 
@@ -158,10 +160,8 @@ protected override async Task OnParametersSetAsync()
         PreviousMeterName = MeterName;
         PreviousInstrumentName = InstrumentName;
 
-        var filters = CreateUpdatedFilters(hasInstrumentChanged);
-
-        DimensionFilters.Clear();
-        DimensionFilters.AddRange(filters);
+        // Replace filters collection on change. Filters can be accessed from a background task so it is immutable for thread safety.
+        DimensionFilters = ImmutableList.Create(CollectionsMarshal.AsSpan(CreateUpdatedFilters(hasInstrumentChanged)));
 
         await UpdateInstrumentDataAsync(_instrument);
     }

src/Aspire.Dashboard/Components/Controls/Chart/ChartContainer.razor.css

@@ -1,7 +1,42 @@
-::deep .tab-label > svg {
+::deep .tab-label > svg {
     margin-right: 2px;
 }
 
 ::deep .metric-tab {
     margin-top: 10px;
 }
+
+.overflow-warning {
+    font-family: var(--body-font);
+    border: 1px solid var(--messagebar-warning-border-color);
+    background-color: var(--messagebar-warning-background-color);
+    color: var(--neutral-foreground-rest);
+    display: grid;
+    grid-template-columns: 24px auto;
+    width: fit-content;
+    align-items: center;
+    min-height: 36px;
+    border-radius: calc(var(--control-corner-radius)* 1px);
+    padding: 0 12px;
+    column-gap: 8px;
+}
+
+.overflow-warning-icon {
+    grid-column: 1;
+    display: flex;
+    justify-content: center;
+}
+
+.overflow-warning-message {
+    grid-column: 2;
+    padding: 10px 0;
+    align-self: center;
+    font-size: 12px;
+    font-weight: 400;
+    line-height: 16px;
+}
+
+.overflow-warning-message .title {
+    font-weight: 600;
+    padding: 0 4px 0 0;
+}

src/Aspire.Dashboard/Components/Controls/Chart/ChartFilters.razor.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections.Immutable;
 using Aspire.Dashboard.Model;
 using Aspire.Dashboard.Otlp.Model;
 using Microsoft.AspNetCore.Components;
@@ -16,7 +17,7 @@ public partial class ChartFilters
     public required InstrumentViewModel InstrumentViewModel { get; set; }
 
     [Parameter, EditorRequired]
-    public required List<DimensionFilterViewModel> DimensionFilters { get; set; }
+    public required ImmutableList<DimensionFilterViewModel> DimensionFilters { get; set; }
 
     public bool ShowCounts { get; set; }
 

src/Aspire.Dashboard/Otlp/Model/OtlpInstrument.cs

@@ -27,6 +27,7 @@ public class OtlpInstrumentData
     public required OtlpInstrumentSummary Summary { get; init; }
     public required List<DimensionScope> Dimensions { get; init; }
     public required Dictionary<string, List<string?>> KnownAttributeValues { get; init; }
+    public required bool HasOverflow { get; init; }
 }
 
 [DebuggerDisplay("Name = {Summary.Name}, Unit = {Summary.Unit}, Type = {Summary.Type}")]
@@ -37,9 +38,17 @@ public class OtlpInstrument
 
     public Dictionary<ReadOnlyMemory<KeyValuePair<string, string>>, DimensionScope> Dimensions { get; } = new(ScopeAttributesComparer.Instance);
     public Dictionary<string, List<string?>> KnownAttributeValues { get; } = new();
+    public bool HasOverflow { get; set; }
 
     public DimensionScope FindScope(RepeatedField<KeyValue> attributes, ref KeyValuePair<string, string>[]? tempAttributes)
     {
+        // See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#overflow-attribute
+        // Inspect attributes before they're merged with parent attributes. "otel.metric.overflow" should be the only attribute.
+        if (!HasOverflow && attributes.Count == 1 && attributes[0].Key == "otel.metric.overflow" && attributes[0].Value.GetString() == "true")
+        {
+            HasOverflow = true;
+        }
+
         // We want to find the dimension scope that matches the attributes, but we don't want to allocate.
         // Copy values to a temporary reusable array.
         //
@@ -101,7 +110,8 @@ public static OtlpInstrument Clone(OtlpInstrument instrument, bool cloneData, Da
         var newInstrument = new OtlpInstrument
         {
             Summary = instrument.Summary,
-            Context = instrument.Context
+            Context = instrument.Context,
+            HasOverflow = instrument.HasOverflow
         };
 
         if (cloneData)

src/Aspire.Dashboard/Otlp/Storage/TelemetryRepository.cs

@@ -1174,13 +1174,15 @@ public List<OtlpInstrumentSummary> GetInstrumentsSummaries(ApplicationKey key)
             {
                 Summary = instrument.Summary,
                 KnownAttributeValues = instrument.KnownAttributeValues,
-                Dimensions = instrument.Dimensions.Values.ToList()
+                Dimensions = instrument.Dimensions.Values.ToList(),
+                HasOverflow = instrument.HasOverflow
             };
         }
         else
         {
             var allDimensions = new List<DimensionScope>();
             var allKnownAttributes = new Dictionary<string, List<string?>>();
+            var hasOverflow = false;
 
             foreach (var instrument in instruments)
             {
@@ -1199,13 +1201,16 @@ public List<OtlpInstrumentSummary> GetInstrumentsSummaries(ApplicationKey key)
                         values = knownAttributeValues.Value.ToList();
                     }
                 }
+
+                hasOverflow = hasOverflow || instrument.HasOverflow;
             }
 
             return new OtlpInstrumentData
             {
                 Summary = instruments[0].Summary,
                 Dimensions = allDimensions,
-                KnownAttributeValues = allKnownAttributes
+                KnownAttributeValues = allKnownAttributes,
+                HasOverflow = hasOverflow
             };
         }
     }