From 50612a7d1467e9b47289882b762f658f4c09c5c6 Mon Sep 17 00:00:00 2001 From: PromKnight Date: Sat, 16 Nov 2024 02:19:04 +0000 Subject: [PATCH] fix: Add filtering logic to exclude certain torrents during DMM scraping - Introduced `WipeSomeTissue` method to filter out torrents with specific criteria. - Updated both batched and unbatched processing methods to apply the new filtering logic before storing torrent info. --- src/Zilean.DmmScraper/Features/Dmm/DmmScraping.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Zilean.DmmScraper/Features/Dmm/DmmScraping.cs b/src/Zilean.DmmScraper/Features/Dmm/DmmScraping.cs index f5b80e6..155d845 100644 --- a/src/Zilean.DmmScraper/Features/Dmm/DmmScraping.cs +++ b/src/Zilean.DmmScraper/Features/Dmm/DmmScraping.cs @@ -107,7 +107,9 @@ await AnsiConsole.Progress() logger.LogInformation("Distinct torrents: {Count}", distinctTorrents.Count); - var finalizedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents); + var parsedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents); + + var finalizedTorrents = parsedTorrents.Where(WipeSomeTissue).ToList(); await torrentInfoService.StoreTorrentInfo(finalizedTorrents); } @@ -150,7 +152,9 @@ await Parallel.ForEachAsync(files, parallelOptions, async (file, ct) => { var distinctTorrents = torrents.DistinctBy(x => x.InfoHash).ToList(); - var finalizedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents); + var parsedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents); + + var finalizedTorrents = parsedTorrents.Where(WipeSomeTissue).ToList(); logger.LogInformation("Parsed {Count} torrents", finalizedTorrents.Count); @@ -185,4 +189,8 @@ private static async IAsyncEnumerable ProcessFileAsync(string yield return torrent; } } + + private static bool WipeSomeTissue(TorrentInfo torrent) => + torrent.RawTitle.Contains(" XXX ", StringComparison.OrdinalIgnoreCase) && + !torrent.ParsedTitle.Contains("XXX", StringComparison.OrdinalIgnoreCase); }