From 440f29ea7dba44856bbcf0bb675e21c587fb3b8b Mon Sep 17 00:00:00 2001 From: martincostello Date: Thu, 18 Jun 2026 16:47:26 +0100 Subject: [PATCH 1/4] [OpenTelemetry] Help diagnose dropped traces - Add diagnostic event when an activity is dropped dur to the local parent not being sampled. - Cover the scenario in the troubleshooting guide. Resolves #7426. --- docs/trace/customizing-the-sdk/README.md | 66 +++++++++++++++++ src/OpenTelemetry/CHANGELOG.md | 5 ++ .../Internal/OpenTelemetrySdkEventSource.cs | 19 +++++ src/OpenTelemetry/Trace/TracerProviderSdk.cs | 6 ++ .../Trace/TracerProviderSdkTests.cs | 72 +++++++++++++++++++ 5 files changed, 168 insertions(+) diff --git a/docs/trace/customizing-the-sdk/README.md b/docs/trace/customizing-the-sdk/README.md index 17ae9898aab..d81f30e43de 100644 --- a/docs/trace/customizing-the-sdk/README.md +++ b/docs/trace/customizing-the-sdk/README.md @@ -386,6 +386,72 @@ probability configured via the `OTEL_TRACES_SAMPLER_ARG` environment variable. Follow [this](../extending-the-sdk/README.md#sampler) document to learn about writing custom samplers. +#### Troubleshooting: spans dropped due to an unsampled parent + +The default sampler is `ParentBased(root=AlwaysOn)`. As the name implies, when a +span has a parent, the `ParentBased` sampler honors the parent's sampling +decision: if the parent was **not** sampled (recorded), the child is dropped as +well. This is the behavior required by the +[specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#parentbased) +and is usually what you want, but it can be surprising when the parent activity +is one you did not create yourself. + +One scenario where this can occur is an ASP.NET Core application that emits custom +spans **without** adding the ASP.NET Core instrumentation to your application: + +```csharp +var activitySource = new ActivitySource("MyCompany.MyProduct"); + +builder.Services.AddOpenTelemetry() + .WithTracing(tracing => tracing + .AddSource("MyCompany.MyProduct") + .AddConsoleExporter()); + +app.MapGet("/hello", () => +{ + using var activity = activitySource.StartActivity("Hello"); + return "Hello, World!"; +}); +``` + +ASP.NET Core creates a `Microsoft.AspNetCore.Hosting.HttpRequestIn` activity for +every request (it is used for request logging and context propagation, even when +no tracing is configured). Because the `Microsoft.AspNetCore.Hosting` +`ActivitySource` has **not** been added to the `TracerProvider`, the SDK never +samples that activity, so it is created with its `Recorded` flag set to `false` +and becomes the parent (`Activity.Current`) of the custom `Hello` activity. The +default `ParentBased` sampler then drops the custom activity because its parent +was not recorded. + +You can confirm this is what is happening by enabling +[self-diagnostics](../../../src/OpenTelemetry/README.md#self-diagnostics) at the +`Verbose` level. The SDK emits an event explaining that the activity was dropped +because its local parent is not recorded, together with the name of the +`ActivitySource`. + +To resolve this, pick whichever option best suites your needs: + +* **Record the parent activity (recommended for ASP.NET Core apps).** Add the + [ASP.NET Core + instrumentation package](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/tree/main/src/OpenTelemetry.Instrumentation.AspNetCore) + (`AddAspNetCoreInstrumentation()`), or at minimum register the source with + `AddSource("Microsoft.AspNetCore.Hosting")`. The request activity then becomes + a recorded root and the custom child is sampled normally. + +* **Use a sampler that does not depend on the parent.** For example + `SetSampler(new AlwaysOnSampler())` records every activity from the registered + sources regardless of the parent's sampling decision. Note this also disables + the honoring of upstream (remote) sampling decisions, so it is generally only + appropriate for development or simple scenarios. + +* **Make the custom activity a root span** by starting it with an explicit + default (empty) parent context so the parent-based decision does not apply: + + ```csharp + using var activity = activitySource.StartActivity( + "Hello", ActivityKind.Internal, parentContext: default); + ``` + ## Context Propagation The OpenTelemetry API exposes a method to obtain the default propagator which is diff --git a/src/OpenTelemetry/CHANGELOG.md b/src/OpenTelemetry/CHANGELOG.md index f769d92e9d0..a9a289758ba 100644 --- a/src/OpenTelemetry/CHANGELOG.md +++ b/src/OpenTelemetry/CHANGELOG.md @@ -9,6 +9,11 @@ Notes](../../RELEASENOTES.md). * Fixed a metric point reclaim data race on CPU ARM architectures. ([#7401](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7401)) +* Added a verbose `OpenTelemetry-Sdk` self-diagnostics event that is emitted + when an activity is dropped because its local (in-process) parent is not + recorded. + ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/issues/TODO)) + ## 1.16.0 Released 2026-Jun-10 diff --git a/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs b/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs index b728102984d..ccfd49d91f6 100644 --- a/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs +++ b/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs @@ -178,6 +178,21 @@ public void ExemplarReservoirException(Exception ex) } } + [NonEvent] + public void ActivityDroppedDueToUnsampledLocalParent(string activityName, string activitySourceName, in ActivityContext parentContext) + { + // Only emit for an in-process (local) parent which was not recorded. + // A remote parent's sampling decision is controlled by the caller and is + // expected to flow through, so it is not reported here. + if (parentContext.TraceId != default && + !parentContext.IsRemote && + (parentContext.TraceFlags & ActivityTraceFlags.Recorded) == 0 && + this.IsEnabled(EventLevel.Verbose, EventKeywords.All)) + { + this.ActivityDroppedDueToUnsampledLocalParent(activityName, activitySourceName); + } + } + [Event(4, Message = "Unknown error in SpanProcessor event '{0}': '{1}'.", Level = EventLevel.Error)] public void SpanProcessorException(string evnt, string ex) => this.WriteEvent(4, evnt, ex); @@ -325,6 +340,10 @@ public void MetricViewException(string source, string ex) public void ExemplarReservoirException(string ex) => this.WriteEvent(57, ex); + [Event(58, Message = "Activity '{0}' from source '{1}' was dropped because its local parent activity is not recorded (sampled). To record these activities, register the parent's instrumentation/ActivitySource or configure a non-parent-based sampler such as the AlwaysOnSampler.", Level = EventLevel.Verbose)] + public void ActivityDroppedDueToUnsampledLocalParent(string activityName, string activitySourceName) + => this.WriteEvent(58, activityName, activitySourceName); + void IConfigurationExtensionsLogger.LogInvalidConfigurationValue(string key, string value) => this.InvalidConfigurationValue(key, value); diff --git a/src/OpenTelemetry/Trace/TracerProviderSdk.cs b/src/OpenTelemetry/Trace/TracerProviderSdk.cs index 055d437e052..740159d2959 100644 --- a/src/OpenTelemetry/Trace/TracerProviderSdk.cs +++ b/src/OpenTelemetry/Trace/TracerProviderSdk.cs @@ -481,6 +481,11 @@ private static ActivitySamplingResult ComputeActivitySamplingResult( SamplingDecision.Drop or _ => PropagateOrIgnoreData(ref options), }; + if (samplingResult.Decision == SamplingDecision.Drop) + { + OpenTelemetrySdkEventSource.Log.ActivityDroppedDueToUnsampledLocalParent(options.Name, options.Source.Name, options.Parent); + } + if (activitySamplingResult > ActivitySamplingResult.PropagationData) { if (samplingResult.AttributesOrNull is { } attributes) @@ -567,6 +572,7 @@ private void RunGetRequestedDataOtherSampler(Activity activity) case SamplingDecision.Drop: activity.IsAllDataRequested = false; activity.ActivityTraceFlags &= ~ActivityTraceFlags.Recorded; + OpenTelemetrySdkEventSource.Log.ActivityDroppedDueToUnsampledLocalParent(activity.DisplayName, activity.Source.Name, parentContext); break; case SamplingDecision.RecordOnly: activity.IsAllDataRequested = true; diff --git a/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs b/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs index a2ef4e2e614..db91806af93 100644 --- a/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs +++ b/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Internal; using OpenTelemetry.Resources; using OpenTelemetry.Resources.Tests; using OpenTelemetry.Tests; @@ -1349,6 +1350,77 @@ public void CheckActivityLinksAddedAfterActivityCreation() Assert.Equal(link2.Context, exportedActivity.Links.ElementAt(1).Context); } + [Fact] + public void SdkEmitsDiagnosticEventWhenActivityDroppedDueToUnsampledLocalParent() + { + using var source = new ActivitySource(Utils.GetCurrentMethodName()); + + // Default sampler is ParentBased(AlwaysOn). + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(source.Name) + .Build(); + + // Simulate the framework-created in-process parent activity. Its source is + // not registered with the provider, so it is never sampled (Recorded == false) + // but it still becomes Activity.Current. + using var parent = new Activity("ParentNotRecorded"); + parent.Start(); + + Assert.False(parent.Recorded); + Assert.False(parent.Context.IsRemote); + + // Create the event listener after the parent has started so its thread + // activity id is the one stamped on the diagnostic event. + using var eventListener = new TestEventListener(OpenTelemetrySdkEventSource.Log); + + using (var child = source.StartActivity("Child")) + { + // The child is dropped because its local parent is not recorded. + Assert.Null(child); + } + + parent.Stop(); + + var droppedEvents = eventListener.Messages.Where( + e => e.EventId == 58 && + (e.Payload?.Count ?? 0) >= 2 && + (e.Payload![0] as string) == "Child" && + (e.Payload![1] as string) == source.Name); + + Assert.Single(droppedEvents); + } + + [Fact] + public void SdkDoesNotEmitUnsampledLocalParentEventForRemoteParent() + { + // A remote parent's sampling decision is controlled by the caller and is + // expected to flow through, so the diagnostic event should not be emitted. + using var source = new ActivitySource(Utils.GetCurrentMethodName()); + + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(source.Name) + .Build(); + + using var eventListener = new TestEventListener(OpenTelemetrySdkEventSource.Log); + + var remoteParent = new ActivityContext( + ActivityTraceId.CreateRandom(), + ActivitySpanId.CreateRandom(), + ActivityTraceFlags.None, + isRemote: true); + + using (var child = source.StartActivity("Child", ActivityKind.Internal, remoteParent)) + { + // Dropped as PropagationData by the default ParentBased sampler. + Assert.False(child?.Recorded ?? false); + } + + var droppedEvents = eventListener.Messages.Where( + e => e.EventId == 58 && (e.Payload?.Count ?? 0) >= 2 && (e.Payload![0] as string) == "Child"); + + Assert.Empty(droppedEvents); + } + public void Dispose() => GC.SuppressFinalize(this); From d5e1f8d7bc5a8f085a97af718e74bd24c68f9049 Mon Sep 17 00:00:00 2001 From: Martin Costello Date: Thu, 18 Jun 2026 17:00:33 +0100 Subject: [PATCH 2/4] [OpenTelemetry] Update CHANGELOG Add PR number. --- src/OpenTelemetry/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenTelemetry/CHANGELOG.md b/src/OpenTelemetry/CHANGELOG.md index a9a289758ba..e2a446f5651 100644 --- a/src/OpenTelemetry/CHANGELOG.md +++ b/src/OpenTelemetry/CHANGELOG.md @@ -12,7 +12,7 @@ Notes](../../RELEASENOTES.md). * Added a verbose `OpenTelemetry-Sdk` self-diagnostics event that is emitted when an activity is dropped because its local (in-process) parent is not recorded. - ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet/issues/TODO)) + ([#7427](https://github.com/open-telemetry/opentelemetry-dotnet/issues/7427)) ## 1.16.0 From 67db5838ce9cc9b09e8d9eda83ca940617c9d2ab Mon Sep 17 00:00:00 2001 From: martincostello Date: Wed, 24 Jun 2026 12:04:45 +0100 Subject: [PATCH 3/4] [OpenTelemetry] Address feedback - Update condition for when to log. - Fix typos. --- docs/trace/customizing-the-sdk/README.md | 2 +- src/OpenTelemetry/CHANGELOG.md | 2 +- src/OpenTelemetry/Trace/TracerProviderSdk.cs | 9 +++-- .../Trace/TracerProviderSdkTests.cs | 36 +++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/docs/trace/customizing-the-sdk/README.md b/docs/trace/customizing-the-sdk/README.md index d81f30e43de..dc8cd241081 100644 --- a/docs/trace/customizing-the-sdk/README.md +++ b/docs/trace/customizing-the-sdk/README.md @@ -429,7 +429,7 @@ You can confirm this is what is happening by enabling because its local parent is not recorded, together with the name of the `ActivitySource`. -To resolve this, pick whichever option best suites your needs: +To resolve this, pick whichever option best suits your needs: * **Record the parent activity (recommended for ASP.NET Core apps).** Add the [ASP.NET Core diff --git a/src/OpenTelemetry/CHANGELOG.md b/src/OpenTelemetry/CHANGELOG.md index 1957afce390..ff7a04fb78b 100644 --- a/src/OpenTelemetry/CHANGELOG.md +++ b/src/OpenTelemetry/CHANGELOG.md @@ -22,7 +22,7 @@ Notes](../../RELEASENOTES.md). * Added a verbose `OpenTelemetry-Sdk` self-diagnostics event that is emitted when an activity is dropped because its local (in-process) parent is not recorded. - ([#7427](https://github.com/open-telemetry/opentelemetry-dotnet/issues/7427)) + ([#7427](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7427)) ## 1.16.0 diff --git a/src/OpenTelemetry/Trace/TracerProviderSdk.cs b/src/OpenTelemetry/Trace/TracerProviderSdk.cs index 740159d2959..a59b23d9519 100644 --- a/src/OpenTelemetry/Trace/TracerProviderSdk.cs +++ b/src/OpenTelemetry/Trace/TracerProviderSdk.cs @@ -481,7 +481,7 @@ private static ActivitySamplingResult ComputeActivitySamplingResult( SamplingDecision.Drop or _ => PropagateOrIgnoreData(ref options), }; - if (samplingResult.Decision == SamplingDecision.Drop) + if (samplingResult.Decision == SamplingDecision.Drop && sampler is ParentBasedSampler) { OpenTelemetrySdkEventSource.Log.ActivityDroppedDueToUnsampledLocalParent(options.Name, options.Source.Name, options.Parent); } @@ -572,7 +572,12 @@ private void RunGetRequestedDataOtherSampler(Activity activity) case SamplingDecision.Drop: activity.IsAllDataRequested = false; activity.ActivityTraceFlags &= ~ActivityTraceFlags.Recorded; - OpenTelemetrySdkEventSource.Log.ActivityDroppedDueToUnsampledLocalParent(activity.DisplayName, activity.Source.Name, parentContext); + + if (this.Sampler is ParentBasedSampler) + { + OpenTelemetrySdkEventSource.Log.ActivityDroppedDueToUnsampledLocalParent(activity.DisplayName, activity.Source.Name, parentContext); + } + break; case SamplingDecision.RecordOnly: activity.IsAllDataRequested = true; diff --git a/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs b/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs index db91806af93..e80672fc86f 100644 --- a/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs +++ b/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs @@ -1421,6 +1421,42 @@ public void SdkDoesNotEmitUnsampledLocalParentEventForRemoteParent() Assert.Empty(droppedEvents); } + [Fact] + public void SdkDoesNotEmitUnsampledLocalParentEventForNonParentBasedSampler() + { + using var source = new ActivitySource(Utils.GetCurrentMethodName()); + + var sampler = new TestSampler + { + SamplingAction = _ => new SamplingResult(SamplingDecision.Drop), + }; + + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(source.Name) + .SetSampler(sampler) + .Build(); + + using var parent = new Activity("ParentNotRecorded"); + parent.Start(); + + Assert.False(parent.Recorded); + Assert.False(parent.Context.IsRemote); + + using var eventListener = new TestEventListener(OpenTelemetrySdkEventSource.Log); + + using (var child = source.StartActivity("Child")) + { + Assert.Null(child); + } + + parent.Stop(); + + var droppedEvents = eventListener.Messages.Where( + e => e.EventId == 58 && (e.Payload?.Count ?? 0) >= 2 && (e.Payload![0] as string) == "Child"); + + Assert.Empty(droppedEvents); + } + public void Dispose() => GC.SuppressFinalize(this); From 1e17e9da4481a047045ef9fd92d8be5d6daf4646 Mon Sep 17 00:00:00 2001 From: Martin Costello Date: Wed, 24 Jun 2026 15:51:39 +0100 Subject: [PATCH 4/4] [OpenTelemetry] Fix source name Fix activity source name for ASP.NET Core. --- docs/trace/customizing-the-sdk/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/trace/customizing-the-sdk/README.md b/docs/trace/customizing-the-sdk/README.md index dc8cd241081..e73a554ebee 100644 --- a/docs/trace/customizing-the-sdk/README.md +++ b/docs/trace/customizing-the-sdk/README.md @@ -435,7 +435,7 @@ To resolve this, pick whichever option best suits your needs: [ASP.NET Core instrumentation package](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/tree/main/src/OpenTelemetry.Instrumentation.AspNetCore) (`AddAspNetCoreInstrumentation()`), or at minimum register the source with - `AddSource("Microsoft.AspNetCore.Hosting")`. The request activity then becomes + `AddSource("Microsoft.AspNetCore")`. The request activity then becomes a recorded root and the custom child is sampled normally. * **Use a sampler that does not depend on the parent.** For example