diff --git a/docs/trace/customizing-the-sdk/README.md b/docs/trace/customizing-the-sdk/README.md index 17ae9898aab..e73a554ebee 100644 --- a/docs/trace/customizing-the-sdk/README.md +++ b/docs/trace/customizing-the-sdk/README.md @@ -386,6 +386,72 @@ probability configured via the `OTEL_TRACES_SAMPLER_ARG` environment variable. Follow [this](../extending-the-sdk/README.md#sampler) document to learn about writing custom samplers. +#### Troubleshooting: spans dropped due to an unsampled parent + +The default sampler is `ParentBased(root=AlwaysOn)`. As the name implies, when a +span has a parent, the `ParentBased` sampler honors the parent's sampling +decision: if the parent was **not** sampled (recorded), the child is dropped as +well. This is the behavior required by the +[specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#parentbased) +and is usually what you want, but it can be surprising when the parent activity +is one you did not create yourself. + +One scenario where this can occur is an ASP.NET Core application that emits custom +spans **without** adding the ASP.NET Core instrumentation to your application: + +```csharp +var activitySource = new ActivitySource("MyCompany.MyProduct"); + +builder.Services.AddOpenTelemetry() + .WithTracing(tracing => tracing + .AddSource("MyCompany.MyProduct") + .AddConsoleExporter()); + +app.MapGet("/hello", () => +{ + using var activity = activitySource.StartActivity("Hello"); + return "Hello, World!"; +}); +``` + +ASP.NET Core creates a `Microsoft.AspNetCore.Hosting.HttpRequestIn` activity for +every request (it is used for request logging and context propagation, even when +no tracing is configured). Because the `Microsoft.AspNetCore.Hosting` +`ActivitySource` has **not** been added to the `TracerProvider`, the SDK never +samples that activity, so it is created with its `Recorded` flag set to `false` +and becomes the parent (`Activity.Current`) of the custom `Hello` activity. The +default `ParentBased` sampler then drops the custom activity because its parent +was not recorded. + +You can confirm this is what is happening by enabling +[self-diagnostics](../../../src/OpenTelemetry/README.md#self-diagnostics) at the +`Verbose` level. The SDK emits an event explaining that the activity was dropped +because its local parent is not recorded, together with the name of the +`ActivitySource`. + +To resolve this, pick whichever option best suits your needs: + +* **Record the parent activity (recommended for ASP.NET Core apps).** Add the + [ASP.NET Core + instrumentation package](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/tree/main/src/OpenTelemetry.Instrumentation.AspNetCore) + (`AddAspNetCoreInstrumentation()`), or at minimum register the source with + `AddSource("Microsoft.AspNetCore")`. The request activity then becomes + a recorded root and the custom child is sampled normally. + +* **Use a sampler that does not depend on the parent.** For example + `SetSampler(new AlwaysOnSampler())` records every activity from the registered + sources regardless of the parent's sampling decision. Note this also disables + the honoring of upstream (remote) sampling decisions, so it is generally only + appropriate for development or simple scenarios. + +* **Make the custom activity a root span** by starting it with an explicit + default (empty) parent context so the parent-based decision does not apply: + + ```csharp + using var activity = activitySource.StartActivity( + "Hello", ActivityKind.Internal, parentContext: default); + ``` + ## Context Propagation The OpenTelemetry API exposes a method to obtain the default propagator which is diff --git a/src/OpenTelemetry/CHANGELOG.md b/src/OpenTelemetry/CHANGELOG.md index 94a4c3ce62a..ff7a04fb78b 100644 --- a/src/OpenTelemetry/CHANGELOG.md +++ b/src/OpenTelemetry/CHANGELOG.md @@ -19,6 +19,11 @@ Notes](../../RELEASENOTES.md). no public API or behavioural change. ([#7146](hhttps://github.com/open-telemetry/opentelemetry-dotnet/pull/7146)) +* Added a verbose `OpenTelemetry-Sdk` self-diagnostics event that is emitted + when an activity is dropped because its local (in-process) parent is not + recorded. + ([#7427](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7427)) + ## 1.16.0 Released 2026-Jun-10 diff --git a/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs b/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs index b728102984d..ccfd49d91f6 100644 --- a/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs +++ b/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs @@ -178,6 +178,21 @@ public void ExemplarReservoirException(Exception ex) } } + [NonEvent] + public void ActivityDroppedDueToUnsampledLocalParent(string activityName, string activitySourceName, in ActivityContext parentContext) + { + // Only emit for an in-process (local) parent which was not recorded. + // A remote parent's sampling decision is controlled by the caller and is + // expected to flow through, so it is not reported here. + if (parentContext.TraceId != default && + !parentContext.IsRemote && + (parentContext.TraceFlags & ActivityTraceFlags.Recorded) == 0 && + this.IsEnabled(EventLevel.Verbose, EventKeywords.All)) + { + this.ActivityDroppedDueToUnsampledLocalParent(activityName, activitySourceName); + } + } + [Event(4, Message = "Unknown error in SpanProcessor event '{0}': '{1}'.", Level = EventLevel.Error)] public void SpanProcessorException(string evnt, string ex) => this.WriteEvent(4, evnt, ex); @@ -325,6 +340,10 @@ public void MetricViewException(string source, string ex) public void ExemplarReservoirException(string ex) => this.WriteEvent(57, ex); + [Event(58, Message = "Activity '{0}' from source '{1}' was dropped because its local parent activity is not recorded (sampled). To record these activities, register the parent's instrumentation/ActivitySource or configure a non-parent-based sampler such as the AlwaysOnSampler.", Level = EventLevel.Verbose)] + public void ActivityDroppedDueToUnsampledLocalParent(string activityName, string activitySourceName) + => this.WriteEvent(58, activityName, activitySourceName); + void IConfigurationExtensionsLogger.LogInvalidConfigurationValue(string key, string value) => this.InvalidConfigurationValue(key, value); diff --git a/src/OpenTelemetry/Trace/TracerProviderSdk.cs b/src/OpenTelemetry/Trace/TracerProviderSdk.cs index 055d437e052..a59b23d9519 100644 --- a/src/OpenTelemetry/Trace/TracerProviderSdk.cs +++ b/src/OpenTelemetry/Trace/TracerProviderSdk.cs @@ -481,6 +481,11 @@ private static ActivitySamplingResult ComputeActivitySamplingResult( SamplingDecision.Drop or _ => PropagateOrIgnoreData(ref options), }; + if (samplingResult.Decision == SamplingDecision.Drop && sampler is ParentBasedSampler) + { + OpenTelemetrySdkEventSource.Log.ActivityDroppedDueToUnsampledLocalParent(options.Name, options.Source.Name, options.Parent); + } + if (activitySamplingResult > ActivitySamplingResult.PropagationData) { if (samplingResult.AttributesOrNull is { } attributes) @@ -567,6 +572,12 @@ private void RunGetRequestedDataOtherSampler(Activity activity) case SamplingDecision.Drop: activity.IsAllDataRequested = false; activity.ActivityTraceFlags &= ~ActivityTraceFlags.Recorded; + + if (this.Sampler is ParentBasedSampler) + { + OpenTelemetrySdkEventSource.Log.ActivityDroppedDueToUnsampledLocalParent(activity.DisplayName, activity.Source.Name, parentContext); + } + break; case SamplingDecision.RecordOnly: activity.IsAllDataRequested = true; diff --git a/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs b/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs index a2ef4e2e614..e80672fc86f 100644 --- a/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs +++ b/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTests.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Internal; using OpenTelemetry.Resources; using OpenTelemetry.Resources.Tests; using OpenTelemetry.Tests; @@ -1349,6 +1350,113 @@ public void CheckActivityLinksAddedAfterActivityCreation() Assert.Equal(link2.Context, exportedActivity.Links.ElementAt(1).Context); } + [Fact] + public void SdkEmitsDiagnosticEventWhenActivityDroppedDueToUnsampledLocalParent() + { + using var source = new ActivitySource(Utils.GetCurrentMethodName()); + + // Default sampler is ParentBased(AlwaysOn). + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(source.Name) + .Build(); + + // Simulate the framework-created in-process parent activity. Its source is + // not registered with the provider, so it is never sampled (Recorded == false) + // but it still becomes Activity.Current. + using var parent = new Activity("ParentNotRecorded"); + parent.Start(); + + Assert.False(parent.Recorded); + Assert.False(parent.Context.IsRemote); + + // Create the event listener after the parent has started so its thread + // activity id is the one stamped on the diagnostic event. + using var eventListener = new TestEventListener(OpenTelemetrySdkEventSource.Log); + + using (var child = source.StartActivity("Child")) + { + // The child is dropped because its local parent is not recorded. + Assert.Null(child); + } + + parent.Stop(); + + var droppedEvents = eventListener.Messages.Where( + e => e.EventId == 58 && + (e.Payload?.Count ?? 0) >= 2 && + (e.Payload![0] as string) == "Child" && + (e.Payload![1] as string) == source.Name); + + Assert.Single(droppedEvents); + } + + [Fact] + public void SdkDoesNotEmitUnsampledLocalParentEventForRemoteParent() + { + // A remote parent's sampling decision is controlled by the caller and is + // expected to flow through, so the diagnostic event should not be emitted. + using var source = new ActivitySource(Utils.GetCurrentMethodName()); + + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(source.Name) + .Build(); + + using var eventListener = new TestEventListener(OpenTelemetrySdkEventSource.Log); + + var remoteParent = new ActivityContext( + ActivityTraceId.CreateRandom(), + ActivitySpanId.CreateRandom(), + ActivityTraceFlags.None, + isRemote: true); + + using (var child = source.StartActivity("Child", ActivityKind.Internal, remoteParent)) + { + // Dropped as PropagationData by the default ParentBased sampler. + Assert.False(child?.Recorded ?? false); + } + + var droppedEvents = eventListener.Messages.Where( + e => e.EventId == 58 && (e.Payload?.Count ?? 0) >= 2 && (e.Payload![0] as string) == "Child"); + + Assert.Empty(droppedEvents); + } + + [Fact] + public void SdkDoesNotEmitUnsampledLocalParentEventForNonParentBasedSampler() + { + using var source = new ActivitySource(Utils.GetCurrentMethodName()); + + var sampler = new TestSampler + { + SamplingAction = _ => new SamplingResult(SamplingDecision.Drop), + }; + + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(source.Name) + .SetSampler(sampler) + .Build(); + + using var parent = new Activity("ParentNotRecorded"); + parent.Start(); + + Assert.False(parent.Recorded); + Assert.False(parent.Context.IsRemote); + + using var eventListener = new TestEventListener(OpenTelemetrySdkEventSource.Log); + + using (var child = source.StartActivity("Child")) + { + Assert.Null(child); + } + + parent.Stop(); + + var droppedEvents = eventListener.Messages.Where( + e => e.EventId == 58 && (e.Payload?.Count ?? 0) >= 2 && (e.Payload![0] as string) == "Child"); + + Assert.Empty(droppedEvents); + } + public void Dispose() => GC.SuppressFinalize(this);