Cleanup data from the InvoiceEvents table

BTCPayServer.Data/ApplicationDbContext.cs

@@ -28,7 +28,6 @@ public ApplicationDbContext(DbContextOptions<ApplicationDbContext> options)
         public DbSet<APIKeyData> ApiKeys { get; set; }
         public DbSet<AppData> Apps { get; set; }
         public DbSet<StoredFile> Files { get; set; }
-        public DbSet<InvoiceEventData> InvoiceEvents { get; set; }
         public DbSet<InvoiceSearchData> InvoiceSearches { get; set; }
         public DbSet<InvoiceWebhookDeliveryData> InvoiceWebhookDeliveries { get; set; }
         public DbSet<InvoiceData> Invoices { get; set; }
@@ -75,7 +74,6 @@ protected override void OnModelCreating(ModelBuilder builder)
             APIKeyData.OnModelCreating(builder, Database);
             AppData.OnModelCreating(builder, Database);
             //StoredFile.OnModelCreating(builder);
-            InvoiceEventData.OnModelCreating(builder);
             InvoiceSearchData.OnModelCreating(builder);
             InvoiceWebhookDeliveryData.OnModelCreating(builder);
             InvoiceData.OnModelCreating(builder, Database);

BTCPayServer.Data/Data/InvoiceData.cs

@@ -16,7 +16,6 @@ public partial class InvoiceData : IHasBlobUntyped
 
         public DateTimeOffset Created { get; set; }
         public List<PaymentData> Payments { get; set; }
-        public List<InvoiceEventData> Events { get; set; }
 
         [Obsolete("Use Blob2 instead")]
         public byte[] Blob { get; set; }

BTCPayServer.Data/Data/InvoiceEventData.cs

@@ -6,28 +6,10 @@ namespace BTCPayServer.Data
     public class InvoiceEventData
     {
         public string InvoiceDataId { get; set; }
-        public InvoiceData InvoiceData { get; set; }
-        public string UniqueId { get; set; }
         public DateTimeOffset Timestamp { get; set; }
         public string Message { get; set; }
         public EventSeverity Severity { get; set; } = EventSeverity.Info;
 
-
-        internal static void OnModelCreating(ModelBuilder builder)
-        {
-            builder.Entity<InvoiceEventData>()
-                   .HasOne(o => o.InvoiceData)
-                   .WithMany(i => i.Events).OnDelete(DeleteBehavior.Cascade);
-            builder.Entity<InvoiceEventData>()
-                .HasKey(o => new
-                {
-                    o.InvoiceDataId,
-#pragma warning disable CS0618
-                    o.UniqueId
-#pragma warning restore CS0618
-                });
-        }
-
         public enum EventSeverity
         {
             Info,

BTCPayServer.Data/Migrations/20240405004015_cleanup_invoice_events.cs

@@ -0,0 +1,34 @@
+using System;
+using BTCPayServer.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace BTCPayServer.Migrations
+{
+    [DbContext(typeof(ApplicationDbContext))]
+    [Migration("20240405004015_cleanup_invoice_events")]
+    public partial class cleanup_invoice_events : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            if (migrationBuilder.IsNpgsql())
+            {
+                migrationBuilder.Sql(@"
+                    ALTER TABLE ""InvoiceEvents"" DROP CONSTRAINT IF EXISTS ""PK_InvoiceEvents"";
+                    ALTER TABLE ""InvoiceEvents"" DROP COLUMN IF EXISTS ""UniqueId"";
+                    CREATE INDEX IF NOT EXISTS ""IX_InvoiceEvents_InvoiceDataId"" ON ""InvoiceEvents""(""InvoiceDataId"");
+                    VACUUM (FULL, ANALYZE) ""InvoiceEvents"";
+                ", true);
+            }
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+        }
+    }
+}

BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs

@@ -305,28 +305,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("Invoices");
                 });
 
-            modelBuilder.Entity("BTCPayServer.Data.InvoiceEventData", b =>
-                {
-                    b.Property<string>("InvoiceDataId")
-                        .HasColumnType("text");
-
-                    b.Property<string>("UniqueId")
-                        .HasColumnType("text");
-
-                    b.Property<string>("Message")
-                        .HasColumnType("text");
-
-                    b.Property<int>("Severity")
-                        .HasColumnType("integer");
-
-                    b.Property<DateTimeOffset>("Timestamp")
-                        .HasColumnType("timestamp with time zone");
-
-                    b.HasKey("InvoiceDataId", "UniqueId");
-
-                    b.ToTable("InvoiceEvents");
-                });
-
             modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b =>
                 {
                     b.Property<int>("Id")
@@ -1244,17 +1222,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.Navigation("StoreData");
                 });
 
-            modelBuilder.Entity("BTCPayServer.Data.InvoiceEventData", b =>
-                {
-                    b.HasOne("BTCPayServer.Data.InvoiceData", "InvoiceData")
-                        .WithMany("Events")
-                        .HasForeignKey("InvoiceDataId")
-                        .OnDelete(DeleteBehavior.Cascade)
-                        .IsRequired();
-
-                    b.Navigation("InvoiceData");
-                });
-
             modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b =>
                 {
                     b.HasOne("BTCPayServer.Data.InvoiceData", "InvoiceData")
@@ -1600,8 +1567,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                 {
                     b.Navigation("AddressInvoices");
 
-                    b.Navigation("Events");
-
                     b.Navigation("InvoiceSearchData");
 
                     b.Navigation("Payments");

BTCPayServer/Controllers/UIInvoiceController.UI.cs

@@ -103,7 +103,6 @@ public async Task<IActionResult> Invoice(string invoiceId)
                 InvoiceId = new[] { invoiceId },
                 UserId = GetUserId(),
                 IncludeAddresses = true,
-                IncludeEvents = true,
                 IncludeArchived = true,
                 IncludeRefunds = true,
             })).FirstOrDefault();
@@ -144,7 +143,7 @@ public async Task<IActionResult> Invoice(string invoiceId)
                 RedirectUrl = invoice.RedirectURL?.AbsoluteUri,
                 TypedMetadata = invoice.Metadata,
                 StatusException = invoice.ExceptionStatus,
-                Events = invoice.Events,
+                Events = await _InvoiceRepository.GetInvoiceLogs(invoice.Id),
                 Metadata = metaData,
                 Archived = invoice.Archived,
                 HasRefund = invoice.Refunds.Any(),
@@ -586,7 +585,6 @@ public async Task<IActionResult> ToggleArchive(string invoiceId)
                 InvoiceId = new[] { invoiceId },
                 UserId = GetUserId(),
                 IncludeAddresses = false,
-                IncludeEvents = false,
                 IncludeArchived = true,
             })).FirstOrDefault();
             if (invoice == null)

BTCPayServer/Data/InvoiceDataExtensions.cs

@@ -50,10 +50,6 @@ public static InvoiceEntity GetBlob(this InvoiceData invoiceData)
             {
                 entity.AvailableAddressHashes = invoiceData.AddressInvoices.Select(a => a.GetAddress() + a.GetPaymentMethodId()).ToHashSet();
             }
-            if (invoiceData.Events != null)
-            {
-                entity.Events = invoiceData.Events.OrderBy(c => c.Timestamp).ToList();
-            }
             if (invoiceData.Refunds != null)
             {
                 entity.Refunds = invoiceData.Refunds.OrderBy(c => c.PullPaymentData.StartDate).ToList();

BTCPayServer/Models/InvoicingModels/InvoiceDetailsModel.cs

@@ -120,7 +120,7 @@ public string Fiat
         }
         public InvoiceMetadata TypedMetadata { get; set; }
         public DateTimeOffset MonitoringDate { get; internal set; }
-        public List<InvoiceEventData> Events { get; internal set; }
+        public InvoiceEventData[] Events { get; internal set; }
         public string NotificationEmail { get; internal set; }
         public Dictionary<string, object> Metadata { get; set; }
         public Dictionary<string, object> ReceiptData { get; set; }

BTCPayServer/Services/Invoices/InvoiceEntity.cs

@@ -501,8 +501,6 @@ private Uri FillPlaceholdersUri(string v)
         public HashSet<string> AvailableAddressHashes { get; set; }
         [JsonProperty]
         public bool ExtendedNotifications { get; set; }
-        [JsonIgnore]
-        public List<InvoiceEventData> Events { get; internal set; }
 
         [JsonProperty]
         public double PaymentTolerance { get; set; }

BTCPayServer/Services/Invoices/InvoiceRepository.cs

@@ -16,6 +16,7 @@
 using NBitcoin;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using Npgsql;
 using Encoders = NBitcoin.DataEncoders.Encoders;
 using InvoiceData = BTCPayServer.Data.InvoiceData;
 
@@ -140,7 +141,7 @@ public async Task<AppData[]> GetAppsTaggingStore(string storeId)
 
         public async Task UpdateInvoice(string invoiceId, UpdateCustomerModel data)
         {
-            retry:
+retry:
             using (var ctx = _applicationDbContextFactory.CreateContext())
             {
                 var invoiceData = await ctx.Invoices.FindAsync(invoiceId);
@@ -169,7 +170,7 @@ public async Task UpdateInvoice(string invoiceId, UpdateCustomerModel data)
 
         public async Task UpdateInvoiceExpiry(string invoiceId, TimeSpan seconds)
         {
-            retry:
+retry:
             await using (var ctx = _applicationDbContextFactory.CreateContext())
             {
                 var invoiceData = await ctx.Invoices.FindAsync(invoiceId);
@@ -199,7 +200,7 @@ async Task InvoiceNeedUpdateEventLater(string invoiceId, TimeSpan expirationIn)
 
         public async Task ExtendInvoiceMonitor(string invoiceId)
         {
-            retry:
+retry:
             using (var ctx = _applicationDbContextFactory.CreateContext())
             {
                 var invoiceData = await ctx.Invoices.FindAsync(invoiceId);
@@ -276,28 +277,33 @@ await context.AddressInvoices.AddAsync(new AddressInvoiceData()
         public async Task AddInvoiceLogs(string invoiceId, InvoiceLogs logs)
         {
             await using var context = _applicationDbContextFactory.CreateContext();
-            foreach (var log in logs.ToList())
+            var db = context.Database.GetDbConnection();
+            var data = logs.ToList().Select(log => new InvoiceEventData()
             {
-                await context.InvoiceEvents.AddAsync(new InvoiceEventData()
-                {
-                    Severity = log.Severity,
-                    InvoiceDataId = invoiceId,
-                    Message = log.Log,
-                    Timestamp = log.Timestamp,
-                    UniqueId = Encoders.Hex.EncodeData(RandomUtils.GetBytes(10))
-                });
-            }
-            await context.SaveChangesAsync().ConfigureAwait(false);
+                Severity = log.Severity,
+                InvoiceDataId = invoiceId,
+                Message = log.Log,
+                Timestamp = log.Timestamp
+            }).ToArray();
+
+            await db.ExecuteAsync(InsertInvoiceEvent, data);
+        }
+
+        public async Task<InvoiceEventData[]> GetInvoiceLogs(string invoiceId)
+        {
+            await using var context = _applicationDbContextFactory.CreateContext();
+            var db = context.Database.GetDbConnection();
+            return (await db.QueryAsync<InvoiceEventData>("SELECT * FROM \"InvoiceEvents\" WHERE \"InvoiceDataId\"=@InvoiceDataId ORDER BY \"Timestamp\"", new { InvoiceDataId = invoiceId })).ToArray();
         }
 
         public Task UpdatePaymentDetails(string invoiceId, IPaymentMethodHandler handler, object details)
         {
             return UpdatePaymentDetails(invoiceId, handler.PaymentMethodId, details is null ? null : JToken.FromObject(details, handler.Serializer));
-        
+
         }
         public async Task UpdatePaymentDetails(string invoiceId, PaymentMethodId paymentMethodId, JToken details)
         {
-            retry:
+retry:
             using (var context = _applicationDbContextFactory.CreateContext())
             {
                 try
@@ -347,7 +353,7 @@ public async Task UpdatePrompt(string invoiceId, PaymentPrompt prompt)
         public async Task NewPaymentPrompt(string invoiceId, PaymentMethodContext paymentPromptContext)
         {
             var prompt = paymentPromptContext.Prompt;
-            retry:
+retry:
             using (var context = _applicationDbContextFactory.CreateContext())
             {
                 var invoice = await context.Invoices.FindAsync(invoiceId);
@@ -399,22 +405,27 @@ public async Task AddPendingInvoiceIfNotPresent(string invoiceId)
             }
         }
 
+        const string InsertInvoiceEvent = "INSERT INTO \"InvoiceEvents\" (\"InvoiceDataId\", \"Severity\", \"Message\", \"Timestamp\") VALUES (@InvoiceDataId, @Severity, @Message, @Timestamp)";
+
         public async Task AddInvoiceEvent(string invoiceId, object evt, InvoiceEventData.EventSeverity severity)
         {
             await using var context = _applicationDbContextFactory.CreateContext();
-            await context.InvoiceEvents.AddAsync(new InvoiceEventData()
-            {
-                Severity = severity,
-                InvoiceDataId = invoiceId,
-                Message = evt.ToString(),
-                Timestamp = DateTimeOffset.UtcNow,
-                UniqueId = Encoders.Hex.EncodeData(RandomUtils.GetBytes(10))
-            });
+            var conn = context.Database.GetDbConnection();
             try
             {
-                await context.SaveChangesAsync();
+                await conn.ExecuteAsync(InsertInvoiceEvent,
+                    new InvoiceEventData()
+                    {
+                        Severity = severity,
+                        InvoiceDataId = invoiceId,
+                        Message = evt.ToString(),
+                        Timestamp = DateTimeOffset.UtcNow
+                    });
+            }
+            catch (Npgsql.NpgsqlException ex) when (ex.SqlState == PostgresErrorCodes.ForeignKeyViolation)
+            {
+                // Invoice does not exists
             }
-            catch (DbUpdateException) { } // Probably the invoice does not exists anymore
         }
 
         public static void AddToTextSearch(ApplicationDbContext context, InvoiceData invoice, params string[] terms)
@@ -448,7 +459,7 @@ await context.Database.GetDbConnection()
         }
         internal async Task UpdateInvoicePrice(string invoiceId, decimal price)
         {
-            retry:
+retry:
             using (var context = _applicationDbContextFactory.CreateContext())
             {
                 var invoiceData = await context.FindAsync<Data.InvoiceData>(invoiceId).ConfigureAwait(false);
@@ -744,7 +755,7 @@ private IQueryable<InvoiceData> GetInvoiceQuery(ApplicationDbContext context, In
 
             if (queryObject.Take != null)
                 query = query.Take(queryObject.Take.Value);
-            
+
             return query;
         }
         public Task<InvoiceEntity[]> GetInvoices(InvoiceQuery queryObject)
@@ -758,8 +769,6 @@ public async Task<InvoiceEntity[]> GetInvoices(InvoiceQuery queryObject, Cancell
             query = query.Include(o => o.Payments);
             if (queryObject.IncludeAddresses)
                 query = query.Include(o => o.AddressInvoices);
-            if (queryObject.IncludeEvents)
-                query = query.Include(o => o.Events);
             if (queryObject.IncludeRefunds)
                 query = query.Include(o => o.Refunds).ThenInclude(refundData => refundData.PullPaymentData);
             var data = await query.AsNoTracking().ToArrayAsync(cancellationToken).ConfigureAwait(false);
@@ -963,8 +972,6 @@ public string[] InvoiceId
             set;
         }
         public bool IncludeAddresses { get; set; }
-
-        public bool IncludeEvents { get; set; }
         public bool IncludeArchived { get; set; } = true;
         public bool IncludeRefunds { get; set; }
         public bool OrderByDesc { get; set; } = true;

BTCPayServer/Services/Reporting/ProductsReportProvider.cs

@@ -33,7 +33,6 @@ public override async Task Query(QueryContext queryContext, CancellationToken ca
         {
             IncludeArchived = true,
             IncludeAddresses = false,
-            IncludeEvents = false,
             IncludeRefunds = false,
             StartDate = queryContext.From,
             EndDate = queryContext.To,

BTCPayServer/Views/UIInvoice/Invoice.cshtml

@@ -579,7 +579,7 @@
                 </section>
             }
 
-            @if (Model.Events is { Count: > 0 })
+            @if (Model.Events is { Length: > 0 })
             {
                 <section class="mt-5 d-print-none">
                     <h3 class="mb-0">Events</h3>