diff --git a/Generation/Converters/Argumentum.AssetConverter/WebBasedGenerator/Cardpen/HarvestManager.cs b/Generation/Converters/Argumentum.AssetConverter/WebBasedGenerator/Cardpen/HarvestManager.cs index f21f6232..0074876b 100644 --- a/Generation/Converters/Argumentum.AssetConverter/WebBasedGenerator/Cardpen/HarvestManager.cs +++ b/Generation/Converters/Argumentum.AssetConverter/WebBasedGenerator/Cardpen/HarvestManager.cs @@ -107,6 +107,15 @@ await Parallel.ForEachAsync(targetLanguages, parallelOptionsCardsetLanguage, asy }); }); + // Issue #613 (Option C — retry serial): after the parallel loop drains, re-attempt each + // failed set serially (degree=1) with backoff. A large set that timed out under high + // parallelism often succeeds when it has CardPen/Playwright to itself. Replaces the bag + // with whatever still fails after the retry pass. Gated by HarvestSetRetryAttempts (>0). + if (Config.HarvestSetRetryAttempts > 0 && !failedSets.IsEmpty) + { + failedSets = await RetryFailedHarvestSetsAsync(failedSets, targetCardSets, harvestDictionary, funcBrowser); + } + if (!failedSets.IsEmpty) { var summary = string.Join("; ", failedSets.Select(f => $"{f.cardSet}/{f.language}")); @@ -121,6 +130,99 @@ await Parallel.ForEachAsync(targetLanguages, parallelOptionsCardsetLanguage, asy return harvestDictionary; } + /// + /// Re-attempts each failed harvest set serially (degree=1) with backoff, after the + /// parallel loop has drained (issue #613, Option C). A set that timed out under high + /// parallelism frequently succeeds when it has the CardPen/Playwright resources to itself. + /// Reuses the exact same call path — which re-renders + /// correctly because a failed set never added its key to the dictionary (ContainsKey guard). + /// Returns a new bag holding only the sets that STILL fail after their retry attempts. + /// + private async Task> RetryFailedHarvestSetsAsync( + ConcurrentBag<(string cardSet, string language, string error)> failed, + CardSetJob[] targetCardSets, + ConcurrentDictionary<(string cardsetName, string language), Func> harvestDictionary, + Func> funcBrowser) + { + if (failed.IsEmpty) return failed; + + var attempts = Math.Max(1, Config.HarvestSetRetryAttempts); + var backoff = TimeSpan.FromSeconds(Math.Max(0, Config.HarvestSetRetryBackoffSeconds)); + var residual = new ConcurrentBag<(string cardSet, string language, string error)>(); + var orderedFailed = failed.OrderBy(f => f.cardSet).ThenBy(f => f.language).ToList(); + + Logger.Log($"[HARVEST-RETRY] Retrying {orderedFailed.Count} failed set(s) serially " + + $"(attempts={attempts}, backoff={backoff.TotalSeconds}s) — issue #613."); + + foreach (var (cardSet, language, _) in orderedFailed) + { + var job = targetCardSets.FirstOrDefault(c => c.Name == cardSet); + if (job == null) + { + residual.Add((cardSet, language, "card-set config not found during retry")); + continue; + } + + var succeeded = await RetryAsync( + () => ProcessLocalizedHarvest(job, language, harvestDictionary, funcBrowser), + attempts, backoff, $"{cardSet}/{language}"); + + if (!succeeded) + { + residual.Add((cardSet, language, $"still failing after {attempts} serial retry attempt(s)")); + } + } + + Logger.Log($"[HARVEST-RETRY] Retry pass complete: {orderedFailed.Count - residual.Count} recovered, " + + $"{residual.Count} still failing (issue #613).", + residual.IsEmpty ? MessageType.Info : MessageType.Problem); + + return residual; + } + + /// + /// Pure retry-with-backoff helper (issue #613). Runs up to + /// times, returning true on the first success. On each + /// failure except the last, waits before retrying. Returns + /// false if every attempt failed (never throws — the last exception is logged and + /// swallowed so the caller's aggregate-error path can report the residual set list). + /// Extracted as a pure helper so the retry/backoff contract is unit-testable without a + /// browser (precedent: ComputeExpectedImageCount). + /// + internal static async Task RetryAsync(Func action, int attempts, TimeSpan backoff, string label = "") + { + if (attempts < 1) attempts = 1; + Exception lastError = null; + for (var attempt = 1; attempt <= attempts; attempt++) + { + try + { + await action(); + if (attempt > 1) + { + Logger.Log($"[HARVEST-RETRY] '{label}' succeeded on attempt {attempt}/{attempts}."); + } + return true; + } + catch (Exception ex) + { + lastError = ex; + if (attempt < attempts) + { + Logger.Log($"[HARVEST-RETRY] '{label}' attempt {attempt}/{attempts} failed: {ex.Message}. " + + $"Backing off {backoff.TotalSeconds}s before retry (issue #613).", MessageType.Problem); + if (backoff > TimeSpan.Zero) + { + await Task.Delay(backoff); + } + } + } + } + Logger.Log($"[HARVEST-RETRY] '{label}' exhausted {attempts} attempt(s); giving up. " + + $"Last error: {lastError?.Message}", MessageType.Problem); + return false; + } + public CardSetJob[] GetTargetCardSets() { var targetCardSets = Config.CardSetDocuments diff --git a/Generation/Converters/Argumentum.AssetConverter/WebBasedGenerator/WebBasedGeneratorConfig.cs b/Generation/Converters/Argumentum.AssetConverter/WebBasedGenerator/WebBasedGeneratorConfig.cs index ab7ca98f..1fece990 100644 --- a/Generation/Converters/Argumentum.AssetConverter/WebBasedGenerator/WebBasedGeneratorConfig.cs +++ b/Generation/Converters/Argumentum.AssetConverter/WebBasedGenerator/WebBasedGeneratorConfig.cs @@ -44,6 +44,23 @@ public class WebBasedGeneratorConfig /// public bool ContinueOnHarvestSetFailure { get; set; } = true; + /// + /// Issue #613 (Option C — retry serial): after the parallel harvest loop drains, each + /// failed set (collected in the failedSets bag by ContinueOnHarvestSetFailure) + /// is re-attempted this many times serially (degree=1, no contention) with a + /// backoff between attempts. A large set that timed out under high parallelism often + /// succeeds when it has the Playwright/CardPen resources to itself. 0 disables the retry + /// pass (failed sets go straight to the aggregate error, #614 behavior). Default 1. + /// + public int HarvestSetRetryAttempts { get; set; } = 1; + + /// + /// Backoff in seconds between serial retry attempts of a failed harvest set (issue #613). + /// Generous default (30s): the root cause is usually CardPen JS still rendering under + /// prior memory/CPU pressure, so an immediate retry would likely re-fail. 0 = no wait. + /// + public int HarvestSetRetryBackoffSeconds { get; set; } = 30; + public int MaxDegreeOfParallelismImages { get; set; } = 3; public int MaxDegreeOfParallelismImageTranslations { get; set; } = 2;