Avoid potential InvalidOperationException in ActivationData when cancelling operations

src/Orleans.Runtime/Catalog/ActivationData.cs

@@ -1,5 +1,6 @@
 #nullable enable
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
@@ -452,28 +453,46 @@ private void CancelPendingOperations()
         lock (this)
         {
             // If the grain is currently activating, cancel that operation.
-            if (_pendingOperations is not { } operations)
+            if (_pendingOperations is not { Count: > 0 } operations)
             {
                 return;
             }
 
-            foreach (var op in operations)
+            var opCount = operations.Count;
+            // We could be using an ArrayPool<Command> here, but we would need to filter out the
+            // non-command types, which means we would need to loop over _pendingOperations, which
+            // defeats the purpose of making a snapshot.
+            // Note: CopyTo does not use the queue's enumerator, so its safe to take a snapshot this way.
+            var array = ArrayPool<object>.Shared.Rent(opCount);
+
+            operations.CopyTo(array, 0);
+
+            var snapshot = new Span<object>(array, 0, opCount);
+
+            try
             {
-                if (op is Command cmd)
+                foreach (var op in snapshot)
                 {
-                    try
-                    {
-                        cmd.Cancel();
-                    }
-                    catch (Exception exception)
+                    if (op is Command cmd)
                     {
-                        if (exception is not ObjectDisposedException)
+                        try
                         {
-                            LogErrorCancellingOperation(_shared.Logger, exception, cmd);
+                            cmd.Cancel();
+                        }
+                        catch (Exception exception)
+                        {
+                            if (exception is not ObjectDisposedException)
+                            {
+                                LogErrorCancellingOperation(_shared.Logger, exception, cmd);
+                            }
                         }
                     }
                 }
             }
+            finally
+            {
+                ArrayPool<object>.Shared.Return(array, true);
+            }
         }
     }