perf(dlq): index wolverine_dead_letters for replay/cleanup scans (GH-3279)

docs/guide/durability/dead-letter-storage.md

@@ -23,6 +23,24 @@ To replay dead lettered messages back to the incoming table, you also have a com
 dotnet run -- storage replay
 ```
 
+## Indexing <Badge type="tip" text="6.16" />
+
+The durability agent's background replay and cleanup cycles both filter the `wolverine_dead_letters`
+table on the `replayable` column (and, when [dead letter expiration](#dead-letter-expiration) is
+enabled, on the `expires` column). On a large dead-letter table these queries would otherwise force a
+full table scan on every cycle. Wolverine automatically provisions indexes for these columns as part
+of the normal schema creation/migration — a partial index scoped to the matching rows on PostgreSQL,
+SQL Server, and SQLite, and a plain index on MySQL and Oracle (which don't support filtered indexes).
+There is nothing to configure.
+
+::: warning
+If you are upgrading with an *already very large* `wolverine_dead_letters` table, be aware that the
+schema migration that adds these indexes runs a normal `CREATE INDEX`, which takes a write-blocking
+lock for the duration (potentially minutes on a multi-GB table). If that matters for your deployment,
+create the indexes ahead of time with `CREATE INDEX CONCURRENTLY` (PostgreSQL) / `WITH (ONLINE = ON)`
+(SQL Server) before rolling out the upgrade.
+:::
+
 ## Introspecting an Endpoint's Dead Letter Destination <Badge type="tip" text="6.9" />
 
 Where an endpoint's dead letters actually go varies by transport and configuration: some endpoints

src/Persistence/MySql/Wolverine.MySql/Schema/DeadLettersTable.cs

@@ -32,9 +32,23 @@ public DeadLettersTable(DurabilitySettings durability, string schemaName) : base
         AddColumn<DateTimeOffset>(DatabaseConstants.SentAt);
         AddColumn<bool>(DatabaseConstants.Replayable);
 
+        // GH-3279: DLQ replay and cleanup both filter on `replayable`. MySQL has no filtered/partial
+        // indexes, so this is a plain index on the column — still far better than the full-table scan
+        // the durability agent's replay cycle otherwise pays on every pass.
+        Indexes.Add(new IndexDefinition($"idx_{DatabaseConstants.DeadLetterTable}_replayable")
+        {
+            Columns = [DatabaseConstants.Replayable]
+        });
+
         if (durability.DeadLetterQueueExpirationEnabled)
         {
             AddColumn<DateTimeOffset>(DatabaseConstants.Expires).AllowNulls();
+
+            // Same story for the expiration sweep, which filters on `expires`.
+            Indexes.Add(new IndexDefinition($"idx_{DatabaseConstants.DeadLetterTable}_expires")
+            {
+                Columns = [DatabaseConstants.Expires]
+            });
         }
     }
 }

src/Persistence/Oracle/Wolverine.Oracle/Schema/DeadLettersTable.cs

@@ -32,9 +32,23 @@ public DeadLettersTable(DurabilitySettings durability, string schemaName) : base
         AddColumn<DateTimeOffset>(DatabaseConstants.SentAt);
         AddColumn<bool>(DatabaseConstants.Replayable);
 
+        // GH-3279: DLQ replay and cleanup both filter on `replayable`. This is a plain b-tree index
+        // (WHERE-clause partial indexes are Oracle 23c+ only), which still turns the durability
+        // agent's replay-cycle full scan into an index lookup.
+        Indexes.Add(new IndexDefinition($"idx_{DatabaseConstants.DeadLetterTable}_replayable")
+        {
+            Columns = [DatabaseConstants.Replayable]
+        });
+
         if (durability.DeadLetterQueueExpirationEnabled)
         {
             AddColumn<DateTimeOffset>(DatabaseConstants.Expires).AllowNulls();
+
+            // Same story for the expiration sweep, which filters on `expires`.
+            Indexes.Add(new IndexDefinition($"idx_{DatabaseConstants.DeadLetterTable}_expires")
+            {
+                Columns = [DatabaseConstants.Expires]
+            });
         }
     }
 }

src/Persistence/PostgresqlTests/DeadLetterTable_index_creation.cs

@@ -0,0 +1,68 @@
+using IntegrationTests;
+using Npgsql;
+using Shouldly;
+using Weasel.Core;
+using Weasel.Postgresql;
+using Wolverine;
+using Wolverine.Postgresql.Schema;
+using Wolverine.RDBMS;
+
+namespace PostgresqlTests;
+
+// GH-3279: the DLQ replay INSERT-SELECT and cleanup DELETE both filter on `replayable`, and the
+// expiration sweep filters on `expires`. Without indexes these seq-scan the whole (potentially
+// multi-GB) dead-letter table on every durability cycle. DeadLettersTable now provisions partial
+// indexes for both. These tests prove the indexes are created AND — critically — that their
+// predicates round-trip through pg_get_indexdef so a subsequent schema migration reports no drift
+// (a mismatched predicate would make Weasel drop+recreate the index on every startup).
+public class DeadLetterTable_index_creation : IAsyncLifetime
+{
+    private NpgsqlConnection theConnection = null!;
+
+    public async Task InitializeAsync()
+    {
+        theConnection = new NpgsqlConnection(Servers.PostgresConnectionString);
+        await theConnection.OpenAsync();
+    }
+
+    public async Task DisposeAsync()
+    {
+        await theConnection.DisposeAsync();
+    }
+
+    [Fact]
+    public async Task creates_the_replayable_index_and_is_stable_without_expiration()
+    {
+        await theConnection.ResetSchemaAsync("dlq_idx_no_exp");
+
+        var durability = new DurabilitySettings { DeadLetterQueueExpirationEnabled = false };
+        var table = new DeadLettersTable(durability, "dlq_idx_no_exp");
+
+        table.Indexes.ShouldContain(x => x.Name.Contains("replayable"));
+        table.Indexes.ShouldNotContain(x => x.Name.Contains("expires"));
+
+        await table.ApplyChangesAsync(theConnection);
+
+        // Re-reading the just-created schema must report NO difference. If the partial-index
+        // predicate did not round-trip, this would come back as Update and thrash on every startup.
+        var delta = await table.FindDeltaAsync(theConnection);
+        delta.Difference.ShouldBe(SchemaPatchDifference.None);
+    }
+
+    [Fact]
+    public async Task creates_replayable_and_expires_indexes_and_is_stable_with_expiration()
+    {
+        await theConnection.ResetSchemaAsync("dlq_idx_exp");
+
+        var durability = new DurabilitySettings { DeadLetterQueueExpirationEnabled = true };
+        var table = new DeadLettersTable(durability, "dlq_idx_exp");
+
+        table.Indexes.ShouldContain(x => x.Name.Contains("replayable"));
+        table.Indexes.ShouldContain(x => x.Name.Contains("expires"));
+
+        await table.ApplyChangesAsync(theConnection);
+
+        var delta = await table.FindDeltaAsync(theConnection);
+        delta.Difference.ShouldBe(SchemaPatchDifference.None);
+    }
+}

src/Persistence/SqlServerTests/DeadLetterTable_index_creation.cs

@@ -0,0 +1,66 @@
+using IntegrationTests;
+using Microsoft.Data.SqlClient;
+using Shouldly;
+using Weasel.Core;
+using Weasel.SqlServer;
+using Wolverine;
+using Wolverine.RDBMS;
+using Wolverine.SqlServer.Schema;
+
+namespace SqlServerTests;
+
+// GH-3279: DeadLettersTable provisions a filtered index on `replayable` (and, when DLQ expiration is
+// enabled, on `expires`) so the durability agent's replay/cleanup queries stop scanning the whole
+// table. These tests prove the indexes are created AND that their filter predicates round-trip
+// through sys.indexes so a subsequent migration reports no drift (SqlServer stores filter
+// definitions like `([replayable]=(1))`, which must canonicalize back to the configured predicate).
+[Collection("sqlserver")]
+public class DeadLetterTable_index_creation : IAsyncLifetime
+{
+    private SqlConnection theConnection = null!;
+
+    public async Task InitializeAsync()
+    {
+        theConnection = new SqlConnection(Servers.SqlServerConnectionString);
+        await theConnection.OpenAsync();
+    }
+
+    public async Task DisposeAsync()
+    {
+        await theConnection.DisposeAsync();
+    }
+
+    [Fact]
+    public async Task creates_the_replayable_index_and_is_stable_without_expiration()
+    {
+        await theConnection.ResetSchemaAsync("dlq_idx_no_exp");
+
+        var durability = new DurabilitySettings { DeadLetterQueueExpirationEnabled = false };
+        var table = new DeadLettersTable(durability, "dlq_idx_no_exp");
+
+        table.Indexes.ShouldContain(x => x.Name.Contains("replayable"));
+        table.Indexes.ShouldNotContain(x => x.Name.Contains("expires"));
+
+        await table.ApplyChangesAsync(theConnection);
+
+        var delta = await table.FindDeltaAsync(theConnection);
+        delta.Difference.ShouldBe(SchemaPatchDifference.None);
+    }
+
+    [Fact]
+    public async Task creates_replayable_and_expires_indexes_and_is_stable_with_expiration()
+    {
+        await theConnection.ResetSchemaAsync("dlq_idx_exp");
+
+        var durability = new DurabilitySettings { DeadLetterQueueExpirationEnabled = true };
+        var table = new DeadLettersTable(durability, "dlq_idx_exp");
+
+        table.Indexes.ShouldContain(x => x.Name.Contains("replayable"));
+        table.Indexes.ShouldContain(x => x.Name.Contains("expires"));
+
+        await table.ApplyChangesAsync(theConnection);
+
+        var delta = await table.FindDeltaAsync(theConnection);
+        delta.Difference.ShouldBe(SchemaPatchDifference.None);
+    }
+}

src/Persistence/SqliteTests/DeadLetterTable_index_creation.cs

@@ -0,0 +1,63 @@
+using Microsoft.Data.Sqlite;
+using Shouldly;
+using Weasel.Core;
+using Weasel.Sqlite;
+using Wolverine;
+using Wolverine.RDBMS;
+using Wolverine.Sqlite.Schema;
+using Xunit;
+
+namespace SqliteTests;
+
+// GH-3279: DeadLettersTable provisions partial indexes on `replayable` (and `expires` when DLQ
+// expiration is enabled) so the durability agent's replay/cleanup queries stop scanning the whole
+// table. These tests prove the indexes are created AND that their predicates round-trip through
+// sqlite_master so a subsequent migration reports no drift.
+public class DeadLetterTable_index_creation : IAsyncLifetime
+{
+    private SqliteTestDatabase _database = null!;
+    private SqliteConnection theConnection = null!;
+
+    public async Task InitializeAsync()
+    {
+        _database = Servers.CreateDatabase(nameof(DeadLetterTable_index_creation));
+        theConnection = new SqliteConnection(_database.ConnectionString);
+        await theConnection.OpenAsync();
+    }
+
+    public async Task DisposeAsync()
+    {
+        await theConnection.DisposeAsync();
+        _database.Dispose();
+    }
+
+    [Fact]
+    public async Task creates_the_replayable_index_and_is_stable_without_expiration()
+    {
+        var durability = new DurabilitySettings { DeadLetterQueueExpirationEnabled = false };
+        var table = new DeadLettersTable(durability, "main");
+
+        table.Indexes.ShouldContain(x => x.Name.Contains("replayable"));
+        table.Indexes.ShouldNotContain(x => x.Name.Contains("expires"));
+
+        await table.ApplyChangesAsync(theConnection);
+
+        var delta = await table.FindDeltaAsync(theConnection);
+        delta.Difference.ShouldBe(SchemaPatchDifference.None);
+    }
+
+    [Fact]
+    public async Task creates_replayable_and_expires_indexes_and_is_stable_with_expiration()
+    {
+        var durability = new DurabilitySettings { DeadLetterQueueExpirationEnabled = true };
+        var table = new DeadLettersTable(durability, "main");
+
+        table.Indexes.ShouldContain(x => x.Name.Contains("replayable"));
+        table.Indexes.ShouldContain(x => x.Name.Contains("expires"));
+
+        await table.ApplyChangesAsync(theConnection);
+
+        var delta = await table.FindDeltaAsync(theConnection);
+        delta.Difference.ShouldBe(SchemaPatchDifference.None);
+    }
+}

src/Persistence/Wolverine.Postgresql/Schema/DeadLettersTable.cs

@@ -1,4 +1,5 @@
 using Weasel.Core;
+using Weasel.Postgresql;
 using Weasel.Postgresql.Tables;
 using Wolverine.RDBMS;
 
@@ -31,11 +32,28 @@ public DeadLettersTable(DurabilitySettings durability, string schemaName) : base
         AddColumn<DateTimeOffset>(DatabaseConstants.SentAt);
         AddColumn<bool>(DatabaseConstants.Replayable);
 
+        // GH-3279: the durability agent's DLQ replay (INSERT ... SELECT ... WHERE replayable = $1)
+        // and cleanup DELETE both filter on `replayable`. Without an index this seq-scans the whole
+        // dead-letter table on every replay cycle — a multi-second, multi-GB scan for the handful of
+        // replayable rows. A partial index scoped to replayable = true stays tiny (replayed rows are
+        // deleted immediately) and turns both statements into index lookups.
+        Indexes.Add(new IndexDefinition(PostgresqlIdentifier.Shorten($"idx_{DatabaseConstants.DeadLetterTable}_replayable"))
+        {
+            Columns = [DatabaseConstants.Replayable],
+            Predicate = $"{DatabaseConstants.Replayable} = true"
+        });
+
         if (durability.DeadLetterQueueExpirationEnabled)
         {
             AddColumn<DateTimeOffset>(DatabaseConstants.Expires).AllowNulls();
+
+            // Same story for the expiration sweep (DeleteExpiredDeadLetterMessagesOperation) which
+            // filters on `expires`. Only present when DLQ expiration is enabled, so the index is too.
+            Indexes.Add(new IndexDefinition(PostgresqlIdentifier.Shorten($"idx_{DatabaseConstants.DeadLetterTable}_expires"))
+            {
+                Columns = [DatabaseConstants.Expires],
+                Predicate = $"{DatabaseConstants.Expires} is not null"
+            });
         }
-
-
     }
 }

src/Persistence/Wolverine.SqlServer/Schema/DeadLettersTable.cs

@@ -30,10 +30,31 @@ public DeadLettersTable(DurabilitySettings durability, string schemaName) : base
 
         AddColumn<DateTimeOffset>(DatabaseConstants.SentAt);
         AddColumn<bool>(DatabaseConstants.Replayable);
-
+
+        // GH-3279: the durability agent's DLQ replay (INSERT ... SELECT ... WHERE replayable = @p)
+        // and cleanup DELETE both filter on `replayable`. Without an index this scans the whole
+        // dead-letter table on every replay cycle. A filtered index scoped to replayable stays tiny
+        // (replayed rows are deleted immediately) and turns both statements into index seeks.
+        // The predicate uses SqlServer's canonical filter form ([col]=val — bracketed, no spaces)
+        // so it round-trips through sys.indexes.filter_definition instead of thrashing (drop+recreate)
+        // on every schema migration.
+        Indexes.Add(new IndexDefinition($"idx_{DatabaseConstants.DeadLetterTable}_replayable")
+        {
+            Columns = [DatabaseConstants.Replayable],
+            Predicate = $"[{DatabaseConstants.Replayable}]=1"
+        });
+
         if (durability.DeadLetterQueueExpirationEnabled)
         {
             AddColumn<DateTimeOffset>(DatabaseConstants.Expires).AllowNulls();
+
+            // Same story for the expiration sweep (DeleteExpiredDeadLetterMessagesOperation) which
+            // filters on `expires`. Only present when DLQ expiration is enabled, so the index is too.
+            Indexes.Add(new IndexDefinition($"idx_{DatabaseConstants.DeadLetterTable}_expires")
+            {
+                Columns = [DatabaseConstants.Expires],
+                Predicate = $"[{DatabaseConstants.Expires}] IS NOT NULL"
+            });
         }
     }
 }

src/Persistence/Wolverine.Sqlite/Schema/DeadLettersTable.cs

@@ -23,6 +23,14 @@ public DeadLettersTable(DurabilitySettings durability, string schemaName) : base
         AddColumn(DatabaseConstants.SentAt, "TEXT").NotNull().DefaultValueByExpression("(datetime('now'))");
         AddColumn(DatabaseConstants.Replayable, "INTEGER").DefaultValue(1);
 
+        // GH-3279: DLQ replay and cleanup both filter on `replayable`; a partial index scoped to the
+        // handful of replayable rows keeps both off a full-table scan.
+        Indexes.Add(new IndexDefinition($"idx_{DatabaseConstants.DeadLetterTable}_replayable")
+        {
+            Columns = [DatabaseConstants.Replayable],
+            Predicate = $"{DatabaseConstants.Replayable} = 1"
+        });
+
         if (durability.DeadLetterQueueExpirationEnabled)
         {
             // GH-3071 — every other RDBMS backend (Postgres, SqlServer, MySql,
@@ -36,6 +44,13 @@ public DeadLettersTable(DurabilitySettings durability, string schemaName) : base
             // `no such column: expires`. Aligning with the rest of the
             // backends closes the schema-vs-SQL gap.
             AddColumn(DatabaseConstants.Expires, "TEXT");
+
+            // Same story for the expiration sweep, which filters on `expires`.
+            Indexes.Add(new IndexDefinition($"idx_{DatabaseConstants.DeadLetterTable}_expires")
+            {
+                Columns = [DatabaseConstants.Expires],
+                Predicate = $"{DatabaseConstants.Expires} is not null"
+            });
         }
     }
 }