fix: dedupe infohashes

iPromKnight · Nov 17, 2024 · caa116d · caa116d
1 parent 4db3ff3
commit caa116d
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/Zilean.Scraper/Features/Ingestion/GenericIngestionProcessor.cs b/src/Zilean.Scraper/Features/Ingestion/GenericIngestionProcessor.cs
@@ -120,7 +120,7 @@ private async Task ProcessBatch(List<Task<StreamedEntry>> batch, CancellationTok
 
             var existingInfoHashes = await torrentInfoService.GetExistingInfoHashesAsync(infoHashes);
 
-            var newTorrents = torrents.Where(t => !existingInfoHashes.Contains(t.InfoHash)).ToList();
+            var newTorrents = torrents.DistinctBy(x => x.InfoHash).Where(t => !existingInfoHashes.Contains(t.InfoHash)).ToList();
             logger.LogInformation("Filtered out {Count} torrents already in the database", torrents.Count - newTorrents.Count);
 
             if (newTorrents.Count == 0)