Handle missing Cosmos reminder deletes

src/Azure/Orleans.Reminders.Cosmos/CosmosReminderTable.cs

@@ -221,7 +221,7 @@ await _executor.ExecuteOperation(static args =>
 
             return true;
         }
-        catch (CosmosException dce) when (dce.StatusCode is HttpStatusCode.PreconditionFailed)
+        catch (CosmosException dce) when (dce.StatusCode is HttpStatusCode.PreconditionFailed or HttpStatusCode.NotFound)
         {
             return false;
         }
@@ -436,4 +436,4 @@ private readonly struct UIntLogValue(uint value)
         Message = "Error deleting Azure Cosmos DB database"
     )]
     private partial void LogErrorDeletingAzureCosmosDBDatabase(Exception ex);
-}
+}

src/Orleans.Reminders/ReminderService/LocalReminderService.cs

@@ -197,6 +197,11 @@ public async Task UnregisterReminder(IGrainReminder reminder)
 
             // remove from persistent/memory store
             var success = await reminderTable.RemoveRow(grainId, reminderName, eTag);
+            if (!success)
+            {
+                success = await IsReminderAlreadyRemoved(grainId, reminderName, reminder);
+            }
+
             if (success)
             {
                 if (localReminders.Remove(new(grainId, reminderName), out var localRem))
@@ -220,6 +225,17 @@ public async Task UnregisterReminder(IGrainReminder reminder)
             }
         }
 
+        private async Task<bool> IsReminderAlreadyRemoved(GrainId grainId, string reminderName, IGrainReminder reminder)
+        {
+            if (await reminderTable.ReadRow(grainId, reminderName) is not null)
+            {
+                return false;
+            }
+
+            LogDebugReminderAlreadyRemoved(reminder);
+            return true;
+        }
+
         private void ObserveLocalReminderStop(Task stopTask, GrainId grainId, string reminderName)
         {
             ArgumentNullException.ThrowIfNull(stopTask);
@@ -903,6 +919,12 @@ private readonly struct ReminderIdentity(GrainId grainId, string reminderName) :
         )]
         private partial void LogRemovedReminderFromTable(IGrainReminder entry);
 
+        [LoggerMessage(
+            Level = LogLevel.Debug,
+            Message = "Reminder was already absent from the reminder table during unregister: {Entry}."
+        )]
+        private partial void LogDebugReminderAlreadyRemoved(IGrainReminder entry);
+
         [LoggerMessage(
             Level = LogLevel.Error,
             EventId = (int)ErrorCode.RS_Unregister_TableError,

test/Extensions/Orleans.Cosmos.Tests/CosmosRemindersTableTests.cs

@@ -0,0 +1,66 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Orleans.Reminders.Cosmos;
+using TestExtensions;
+using UnitTests;
+using UnitTests.RemindersTest;
+using Xunit;
+
+namespace Tester.Cosmos.Reminders;
+
+/// <summary>
+/// Tests for operation of the Orleans reminders table using Azure Cosmos DB.
+/// </summary>
+[TestCategory("Reminders"), TestCategory("Cosmos")]
+public class CosmosRemindersTableTests : ReminderTableTestsBase
+{
+    public CosmosRemindersTableTests(ConnectionStringFixture fixture, TestEnvironmentFixture environment)
+        : base(fixture, environment, CreateFilters())
+    {
+        CosmosTestUtils.CheckCosmosStorage();
+    }
+
+    private static LoggerFilterOptions CreateFilters()
+    {
+        var filters = new LoggerFilterOptions();
+        filters.AddFilter(typeof(CosmosReminderTable).FullName, LogLevel.Trace);
+        return filters;
+    }
+
+    protected override IReminderTable CreateRemindersTable()
+    {
+        CosmosTestUtils.CheckCosmosStorage();
+
+        var options = new CosmosReminderTableOptions();
+        options.ConfigureTestDefaults();
+        return new CosmosReminderTable(loggerFactory, this.ClusterFixture.Services, Options.Create(options), this.clusterOptions);
+    }
+
+    protected override Task<string> GetConnectionString()
+    {
+        return Task.FromResult(TestDefaultConfiguration.CosmosDBAccountKey);
+    }
+
+    [SkippableFact, TestCategory("Functional")]
+    public void RemindersTable_Cosmos_Init()
+    {
+    }
+
+    [SkippableFact, TestCategory("Functional")]
+    public async Task RemindersTable_Cosmos_RemindersRange()
+    {
+        await RemindersRange(50);
+    }
+
+    [SkippableFact, TestCategory("Functional")]
+    public async Task RemindersTable_Cosmos_RemindersParallelUpsert()
+    {
+        await RemindersParallelUpsert();
+    }
+
+    [SkippableFact, TestCategory("Functional")]
+    public async Task RemindersTable_Cosmos_ReminderSimple()
+    {
+        await ReminderSimple();
+    }
+}