From 57fbd112df2b828f2cac2e55a6370f8bb6f391c3 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Wed, 26 Aug 2026 07:57:11 -0500 Subject: [PATCH] GH-4109: report the top three RSS growths, and stop calling them retainers The measurement asked for in the issue is done, and it exonerates the test. use_iasync_enumerable_as_return_value is not a memory problem. Solo in a fresh process it peaks at 397 MB against a 162 MB floor for a trivial host-building test, and the test itself sends three empty records and awaits two 50ms delays. Five consecutive runs of the same unchanged suite named five DIFFERENT tests as the top entry -- +535 MB, +685 MB, +692 MB, +770 MB, +1084 MB -- while peak worker RSS stayed flat at 4956-5468 MB. The issue's third hypothesis was the right one; its counter-argument ("no other test came close") was true of one run only. So the number is real but the attribution is not. A start-to-verdict RSS delta is heap EXPANSION, not live bytes: the GC does not return memory to the OS promptly, so whichever test is running when the heap grows is charged for all of it. Naming one winner "top retainer" reads as an accusation, and the first measurement this ever produced cost a full investigation of an innocent test. Reporting the top three, labelled "largest RSS growth during", makes the shape visible in the log itself: a test that really does retain holds its place across runs, and a run whose three entries are all strangers is saying the ranking means nothing this time. The first run under the new format already shows the difference -- Bug_42_concurrent_creation_of_command_handlers.try_to_break leads at +1171 MB, and it also led a previous run at +1084 MB, which is the only name so far to repeat. What this does NOT settle is the aggregate: CoreTests' worker peaks near 5 GB on every run, and that number is stable where the per-test ranking is not. That is the real question and it wants its own issue. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxbBpN6PmRB5CMSSTdGNr3 --- build/SupervisedTests.cs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/build/SupervisedTests.cs b/build/SupervisedTests.cs index 2243235f6..8bb5b8c15 100644 --- a/build/SupervisedTests.cs +++ b/build/SupervisedTests.cs @@ -291,14 +291,30 @@ bool report(string projectName, string framework, SupervisorResults results, boo Log.Warning(" [STALLED] {Test} — {Seconds}s in flight on lane {Lane}", stalled.DisplayName, (int)stalled.InFlight.TotalSeconds, stalled.Worker.Lane); + // GH-4109. Report the top THREE, and do not call them retainers. + // + // This is a start-to-verdict RSS delta, which is heap EXPANSION, not live bytes: the GC does + // not hand memory back to the OS promptly, so whichever test happens to be running when the + // heap grows is charged for all of it. Naming a single winner "top retainer" reads as an + // accusation against that test, and the very first measurement this ever produced sent us + // after CoreTests.Acceptance.wolverine_as_command_bus.use_iasync_enumerable_as_return_value + // at +685 MB -- a test that sends three empty records and awaits two 50ms delays. + // + // Five consecutive runs of the same unchanged suite named five DIFFERENT tests (+535 MB to + // +1084 MB) while peak RSS stayed flat at 4956-5468 MB. The leader is noise; the peak is the + // signal. Three entries make that visible in the log itself -- a test that really does retain + // will hold its place across runs, and a run whose three entries are all strangers is telling + // you the ranking means nothing this time. var memory = RunResources.For(results); if (memory.IsMeasured) { - Log.Information(" peak worker RSS {Peak}; top retainer: {Retainer}", + var growth = memory.TopRetainers(3) + .Select(x => $"{RunResources.Delta(x.RetainedBytes!.Value)} {x.DisplayName}") + .ToArray(); + + Log.Information(" peak worker RSS {Peak}; largest RSS growth during: {Growth}", RunResources.Humanize(memory.PeakBytes!.Value), - memory.TopRetainers(1) is [{ } top] - ? $"{RunResources.Delta(top.RetainedBytes!.Value)} {top.DisplayName}" - : "(none attributed)"); + growth.Length > 0 ? string.Join(", ", growth) : "(none attributed)"); } foreach (var test in results.Indeterminate)