Additional tracing spans for activation, persistence, and migration

src/Orleans.Core.Abstractions/Diagnostics/ActivitySources.cs

@@ -0,0 +1,32 @@
+using System.Diagnostics;
+
+namespace Orleans.Diagnostics;
+
+public static class ActivitySources
+{
+    /// <summary>
+    /// Spans triggered from application level code
+    /// </summary>
+    public const string ApplicationGrainActivitySourceName = "Microsoft.Orleans.Application";
+    /// <summary>
+    /// Spans triggered from Orleans runtime code
+    /// </summary>
+    public const string RuntimeActivitySourceName = "Microsoft.Orleans.Runtime";
+    /// <summary>
+    /// Spans tied to lifecycle operations such as activation, migration, and deactivation.
+    /// </summary>
+    public const string LifecycleActivitySourceName = "Microsoft.Orleans.Lifecycle";
+    /// <summary>
+    /// Spans tied to persistent storage operations.
+    /// </summary>
+    public const string StorageActivitySourceName = "Microsoft.Orleans.Storage";
+    /// <summary>
+    /// A wildcard name to match all Orleans activity sources.
+    /// </summary>
+    public const string AllActivitySourceName = "Microsoft.Orleans.*";
+
+    internal static readonly ActivitySource ApplicationGrainSource = new(ApplicationGrainActivitySourceName, "1.1.0");
+    internal static readonly ActivitySource RuntimeGrainSource = new(RuntimeActivitySourceName, "2.0.0");
+    internal static readonly ActivitySource LifecycleGrainSource = new(LifecycleActivitySourceName, "1.0.0");
+    internal static readonly ActivitySource StorageGrainSource = new(StorageActivitySourceName, "1.0.0");
+}

src/Orleans.Core.Abstractions/Diagnostics/ActivityTagKeys.cs

@@ -0,0 +1,148 @@
+namespace Orleans.Diagnostics;
+
+/// <summary>
+/// Contains constants for Activity tag keys used throughout Orleans.
+/// </summary>
+internal static class ActivityTagKeys
+{
+    /// <summary>
+    /// The request ID for an async enumerable operation.
+    /// </summary>
+    public const string AsyncEnumerableRequestId = "orleans.async_enumerable.request_id";
+
+    /// <summary>
+    /// The activation ID tag key.
+    /// </summary>
+    public const string ActivationId = "orleans.activation.id";
+
+    /// <summary>
+    /// The activation cause tag key (e.g., "new" or "rehydrate").
+    /// </summary>
+    public const string ActivationCause = "orleans.activation.cause";
+
+    /// <summary>
+    /// The deactivation reason tag key.
+    /// </summary>
+    public const string DeactivationReason = "orleans.deactivation.reason";
+
+    /// <summary>
+    /// The grain ID tag key.
+    /// </summary>
+    public const string GrainId = "orleans.grain.id";
+
+    /// <summary>
+    /// The grain type tag key.
+    /// </summary>
+    public const string GrainType = "orleans.grain.type";
+
+    /// <summary>
+    /// The grain type tag key.
+    /// </summary>
+    public const string GrainState = "orleans.grain.state";
+
+    /// <summary>
+    /// The silo ID tag key.
+    /// </summary>
+    public const string SiloId = "orleans.silo.id";
+
+    /// <summary>
+    /// The directory previous registration present tag key.
+    /// </summary>
+    public const string DirectoryPreviousRegistrationPresent = "orleans.directory.previousRegistration.present";
+
+    /// <summary>
+    /// The directory registered address tag key.
+    /// </summary>
+    public const string DirectoryRegisteredAddress = "orleans.directory.registered.address";
+
+    /// <summary>
+    /// The directory forwarding address tag key.
+    /// </summary>
+    public const string DirectoryForwardingAddress = "orleans.directory.forwarding.address";
+
+    /// <summary>
+    /// The exception type tag key.
+    /// </summary>
+    public const string ExceptionType = "exception.type";
+
+    /// <summary>
+    /// The exception message tag key.
+    /// </summary>
+    public const string ExceptionMessage = "exception.message";
+
+    /// <summary>
+    /// The placement filter type tag key.
+    /// </summary>
+    public const string PlacementFilterType = "orleans.placement.filter.type";
+
+    /// <summary>
+    /// The storage provider tag key.
+    /// </summary>
+    public const string StorageProvider = "orleans.storage.provider";
+
+    /// <summary>
+    /// The storage state name tag key.
+    /// </summary>
+    public const string StorageStateName = "orleans.storage.state.name";
+
+    /// <summary>
+    /// The storage state type tag key.
+    /// </summary>
+    public const string StorageStateType = "orleans.storage.state.type";
+
+    /// <summary>
+    /// The RPC system tag key.
+    /// </summary>
+    public const string RpcSystem = "rpc.system";
+
+    /// <summary>
+    /// The RPC service tag key.
+    /// </summary>
+    public const string RpcService = "rpc.service";
+
+    /// <summary>
+    /// The RPC method tag key.
+    /// </summary>
+    public const string RpcMethod = "rpc.method";
+
+    /// <summary>
+    /// The RPC Orleans target ID tag key.
+    /// </summary>
+    public const string RpcOrleansTargetId = "rpc.orleans.target_id";
+
+    /// <summary>
+    /// The RPC Orleans source ID tag key.
+    /// </summary>
+    public const string RpcOrleansSourceId = "rpc.orleans.source_id";
+
+    /// <summary>
+    /// The exception stacktrace tag key.
+    /// </summary>
+    public const string ExceptionStacktrace = "exception.stacktrace";
+
+    /// <summary>
+    /// The exception escaped tag key.
+    /// </summary>
+    public const string ExceptionEscaped = "exception.escaped";
+
+    /// <summary>
+    /// Indicates whether a rehydration attempt was ignored.
+    /// </summary>
+    public const string RehydrateIgnored = "orleans.rehydrate.ignored";
+
+    /// <summary>
+    /// The reason why a rehydration attempt was ignored.
+    /// </summary>
+    public const string RehydrateIgnoredReason = "orleans.rehydrate.ignored.reason";
+
+    /// <summary>
+    /// The previous registration address during rehydration.
+    /// </summary>
+    public const string RehydratePreviousRegistration = "orleans.rehydrate.previousRegistration";
+
+    /// <summary>
+    /// The target silo address for migration.
+    /// </summary>
+    public const string MigrationTargetSilo = "orleans.migration.target.silo";
+}
+

src/Orleans.Core.Abstractions/Diagnostics/OpenTelemetryHeaders.cs

@@ -0,0 +1,7 @@
+namespace Orleans.Diagnostics;
+
+internal static class OpenTelemetryHeaders
+{
+    internal const string TraceParent = "traceparent";
+    internal const string TraceState = "tracestate";
+}

src/Orleans.Core.Abstractions/Properties/AssemblyInfo.cs

@@ -12,6 +12,7 @@
 [assembly: InternalsVisibleTo("ServiceBus.Tests")]
 [assembly: InternalsVisibleTo("Tester.AzureUtils")]
 [assembly: InternalsVisibleTo("AWSUtils.Tests")]
+[assembly: InternalsVisibleTo("Tester")]
 [assembly: InternalsVisibleTo("TesterInternal")]
 [assembly: InternalsVisibleTo("TestInternalGrainInterfaces")]
 [assembly: InternalsVisibleTo("TestInternalGrains")]

src/Orleans.Core.Abstractions/Runtime/AsyncEnumerableRequest.cs

@@ -9,6 +9,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Orleans.Concurrency;
+using Orleans.Diagnostics;
 using Orleans.Invocation;
 using Orleans.Serialization.Invocation;
 
@@ -199,6 +200,7 @@ internal sealed class AsyncEnumeratorProxy<T> : IAsyncEnumerator<T>
     private int _batchIndex;
     private bool _disposed;
     private bool _isInitialized;
+    private Activity? _sessionActivity;
 
     private bool IsBatch => (_current.State & EnumerationResult.Batch) != 0;
     private bool IsElement => (_current.State & EnumerationResult.Element) != 0;
@@ -270,6 +272,9 @@ public async ValueTask DisposeAsync()
 
         if (_isInitialized)
         {
+            // Restore the session activity as the current activity so that DisposeAsync RPC is parented to it
+            var previousActivity = Activity.Current;
+            Activity.Current = _sessionActivity;
             try
             {
                 await _target.DisposeAsync(_requestId);
@@ -279,9 +284,18 @@ public async ValueTask DisposeAsync()
                 var logger = ((GrainReference)_target).Shared.ServiceProvider.GetRequiredService<ILogger<AsyncEnumerableRequest<T>>>();
                 logger.LogWarning(exception, "Failed to dispose async enumerator.");
             }
+            finally
+            {
+                Activity.Current = previousActivity;
+            }
         }
 
         _cancellationTokenSource?.Dispose();
+
+        // Stop the session activity after DisposeAsync completes
+        _sessionActivity?.Stop();
+        _sessionActivity?.Dispose();
+
         _disposed = true;
     }
 
@@ -302,6 +316,14 @@ public async ValueTask<bool> MoveNextAsync()
         }
 
         var isActive = _isInitialized;
+
+        // Restore the session activity as the current activity so that RPC calls are parented to it
+        var previousActivity = Activity.Current;
+        if (_sessionActivity is not null)
+        {
+            Activity.Current = _sessionActivity;
+        }
+
         try
         {
             (EnumerationResult Status, object Value) result;
@@ -311,6 +333,11 @@ public async ValueTask<bool> MoveNextAsync()
 
                 if (!_isInitialized)
                 {
+                    // Start the session activity on first enumeration call
+                    // This span wraps the entire enumeration session
+                    _sessionActivity = ActivitySources.ApplicationGrainSource.StartActivity(_request.GetActivityName(), ActivityKind.Client);
+                    _sessionActivity?.SetTag(ActivityTagKeys.AsyncEnumerableRequestId, _requestId.ToString());
+
                     // Assume the enumerator is active as soon as the call begins.
                     isActive = true;
                     result = await _target.StartEnumeration(_requestId, _request, _cancellationToken);
@@ -324,10 +351,12 @@ public async ValueTask<bool> MoveNextAsync()
                 isActive = result.Status.IsActive();
                 if (result.Status is EnumerationResult.Error)
                 {
+                    _sessionActivity?.SetStatus(ActivityStatusCode.Error);
                     ExceptionDispatchInfo.Capture((Exception)result.Value).Throw();
                 }
                 else if (result.Status is EnumerationResult.Canceled)
                 {
+                    _sessionActivity?.SetStatus(ActivityStatusCode.Error, "Canceled");
                     throw new OperationCanceledException();
                 }
 
@@ -339,6 +368,7 @@ public async ValueTask<bool> MoveNextAsync()
 
             if (result.Status is EnumerationResult.MissingEnumeratorError)
             {
+                _sessionActivity?.SetStatus(ActivityStatusCode.Error, "MissingEnumerator");
                 throw new EnumerationAbortedException("Enumeration aborted: the remote target does not have a record of this enumerator."
                     + " This likely indicates that the remote grain was deactivated since enumeration begun or that the enumerator was idle for longer than the expiration period.");
             }
@@ -356,6 +386,11 @@ await _target.DisposeAsync(_requestId).AsTask()
                 .ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing);
             throw;
         }
+        finally
+        {
+            // Restore the previous activity after each call
+            Activity.Current = previousActivity;
+        }
     }
 }
 

src/Orleans.Core/Diagnostics/ActivityNames.cs

@@ -0,0 +1,18 @@
+namespace Orleans.Runtime;
+
+public static class ActivityNames
+{
+    public const string PlaceGrain = "place grain";
+    public const string FilterPlacementCandidates = "filter placement candidates";
+    public const string ActivateGrain = "activate grain";
+    public const string DeactivateGrain = "deactivate grain";
+    public const string OnActivate = "execute OnActivateAsync";
+    public const string OnDeactivate = "execute OnDeactivateAsync";
+    public const string RegisterDirectoryEntry = "register directory entry";
+    public const string StorageRead = "read storage";
+    public const string StorageWrite = "write storage";
+    public const string StorageClear = "clear storage";
+    public const string ActivationDehydrate = "dehydrate activation";
+    public const string ActivationRehydrate = "rehydrate activation";
+    public const string WaitMigration = "wait migration";
+}