From 8b5ee56fec52af3da9332806f73fa9c93826412b Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Tue, 21 Jul 2026 10:47:27 -0500
Subject: [PATCH 01/21] Add recycle_on_failure and retries options to runtests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A test that corrupts process-wide state — the motivating case is a GPU
whose driver ends up in a state where every subsequent allocation in the
process fails — poisons every later test scheduled onto the same worker,
turning one bad test into a cascade of failed files. The Distributed-
based harness this package was extracted from recycled a worker after
any failed test; restore that behavior behind `recycle_on_failure =
true`, alongside the existing max-rss and crash recycling.

With `retries = N`, tests that did not pass are re-run up to N times
after the main run completes: sequentially, on a single fresh worker,
with all other workers stopped. Parallel test runs create resource
contention (several workers sharing one GPU or a limited amount of RAM),
so a failure can mean "lost the resource race" rather than "broken":
re-running on an otherwise-idle system distinguishes the two. Tests that
failed due to contention reliably pass on the idle retry, while
deterministic failures fail again and are reported exactly once — only
the final attempt of each test enters the results, and retried tests are
visibly marked in the output.

Both options default to off.
---
 src/ParallelTestRunner.jl | 73 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 8beaba2..1a39c9c 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -998,6 +998,17 @@ runtests(MyPackage, ARGS; serial=["big_alloc_test", "huge_matrix"])
 
 Workers are automatically recycled when they exceed memory limits to prevent out-of-memory
 issues during long test runs. The memory limit is set based on system architecture.
+
+## Failure Handling
+
+With `recycle_on_failure = true`, a worker is recycled after any test that did not pass, so
+a test that corrupts process-wide state (e.g. wedges a GPU driver) cannot poison subsequent
+tests on the same worker.
+
+With `retries = N` (default 0), tests that did not pass are re-run up to `N` times after
+the main run completes — sequentially, on a single fresh worker, with all other workers
+stopped — so tests that failed due to resource pressure from concurrent workers get an
+otherwise-idle system. Only the final attempt of each test is reported.
 """
 function runtests(mod::Module, args::ParsedArgs;
                   testsuite::Dict{String,Expr} = find_tests(pwd()),
@@ -1012,6 +1023,8 @@ function runtests(mod::Module, args::ParsedArgs;
                   stdout = Base.stdout,
                   stderr = Base.stderr,
                   max_worker_rss = get_max_worker_rss(),
+                  recycle_on_failure::Bool = false,
+                  retries::Integer = 0,
                   )
     #
     # set-up
@@ -1069,6 +1082,8 @@ function runtests(mod::Module, args::ParsedArgs;
         stdout,
         stderr,
         max_worker_rss,
+        recycle_on_failure,
+        retries,
     )
 end
 
@@ -1091,6 +1106,8 @@ function _runtests(mod::Module, args::ParsedArgs;
                    stdout = Base.stdout,
                    stderr = Base.stderr,
                    max_worker_rss = get_max_worker_rss(),
+                   recycle_on_failure::Bool = false,
+                   retries::Integer = 0,
                    )
 
     # partition into serial and parallel groups
@@ -1314,6 +1331,7 @@ function _runtests(mod::Module, args::ParsedArgs;
     #
 
     tests_to_start = Threads.Atomic{Int}(length(tests))
+    interrupted = false
     # After parallel-before-serial: stop extra workers so only one process is alive for
     # serial tests, but keep one parallel worker so we do not add a third addworker (ID_COUNTER).
     function drain_pool_leaving_one_worker!(pool, njobs)
@@ -1411,6 +1429,11 @@ function _runtests(mod::Module, args::ParsedArgs;
                                   # the worker has reached the max-rss limit, recycle it
                                   # so future tests start with a smaller working set
                                   Malt.stop(wrkr)
+                              elseif recycle_on_failure && anynonpass(result[])
+                                  # a failing test may have left the worker in a bad state
+                                  # (e.g. a wedged GPU driver whose every later allocation
+                                  # fails); recycle it so future tests get a fresh process
+                                  Malt.stop(wrkr)
                               end
                           else
                               # One of Malt.TerminatedWorkerException, Malt.RemoteException, or ErrorException
@@ -1477,6 +1500,7 @@ function _runtests(mod::Module, args::ParsedArgs;
             end
         end
     catch err
+        interrupted = true
         if !(err isa InterruptException)
             println(io_ctx.stderr, "\nCaught an error, stopping...")
         end
@@ -1514,6 +1538,55 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
     end
 
+    # retry failed tests, if requested: sequentially, on a single fresh worker, with every
+    # other worker gone — tests that failed due to resource pressure (e.g. GPU memory
+    # oversubscription from concurrent workers) reliably pass on an otherwise-idle system.
+    # only the retried result is reported; persistent failures fail again and are reported
+    # exactly once.
+    if retries > 0 && !interrupted && args.quickfail === nothing
+        local retry_wrkr = nothing
+        for round in 1:retries
+            retryable = [r.test for r in results.value
+                         if r.result isa Exception || anynonpass(r.result[])]
+            isempty(retryable) && break
+            println(io_ctx.stdout)
+            printstyled(io_ctx.stdout,
+                        "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
+                        color = :yellow)
+            for test in retryable
+                if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
+                    retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
+                                           exeflags, env)
+                end
+                test_t0 = time()
+                result = try
+                    Malt.remote_eval_wait(Main, retry_wrkr.w, :(import ParallelTestRunner))
+                    Malt.remote_call_fetch(invokelatest, retry_wrkr.w, runtest,
+                                           RecordType, testsuite[test], test,
+                                           init_code, test_t0, custom_args)
+                catch ex
+                    isa(ex, InterruptException) && rethrow()
+                    ex
+                end
+                test_t1 = time()
+                output = @lock retry_wrkr.io String(take!(retry_wrkr.io[]))
+                filter!(r -> r.test != test, results.value)
+                push!(results.value, (; test, result, output, test_t0, test_t1))
+                if result isa AbstractTestRecord && !anynonpass(result[])
+                    printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
+                else
+                    printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
+                    # don't let a failure contaminate the next retry
+                    Malt.stop(retry_wrkr)
+                    retry_wrkr = nothing
+                end
+            end
+        end
+        if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr)
+            Malt.stop(retry_wrkr)
+        end
+    end
+
     # print the output generated by each testset
     # (`@sync` above joined all writers, so `results` is quiescent from here on)
     for (testname, result, output, _start, _stop) in results.value

From a1ed92497dd486da9bf5ff3311070493ece7f730 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Mon, 3 Aug 2026 11:07:56 -0500
Subject: [PATCH 02/21] Document recycle_on_failure and retries

Add a "Failure Handling" section to the advanced usage guide covering both
options: why worker recycling after a failure is useful (process-wide state
corruption cascading onto later tests on the same worker) and what the retry
environment guarantees (all other workers stopped, sequential re-run on a
fresh worker, only the final attempt reported, retry worker recycled after a
repeat failure).

Also mention them in the feature list on the front page, and add a best
practice warning against using retries to paper over genuinely broken tests.
---
 docs/src/advanced.md | 51 ++++++++++++++++++++++++++++++++++++++++++++
 docs/src/index.md    |  9 ++++++++
 2 files changed, 60 insertions(+)

diff --git a/docs/src/advanced.md b/docs/src/advanced.md
index 2ac2e93..b749b7d 100644
--- a/docs/src/advanced.md
+++ b/docs/src/advanced.md
@@ -174,6 +174,55 @@ duration, longest first) and their results appear in the same overall summary.
     If the user filters tests via positional arguments (e.g. `julia test/runtests.jl unit`),
     any serial test names that were filtered out are silently removed from the serial list.
 
+## Failure Handling
+
+Both options described in this section are opt-in and default to off.
+
+### Recycling Workers after a Failure
+
+Workers are reused across tests, so a test that corrupts process-wide state — a wedged GPU driver whose every subsequent allocation fails, a global left in an inconsistent state, a library put in an unusable configuration — can make every later test scheduled on that same worker fail too.
+
+Setting `recycle_on_failure=true` stops the worker after any test that did not pass, so the next test gets a fresh process:
+
+```julia
+runtests(MyPackage, ARGS; recycle_on_failure=true)
+```
+
+This complements the existing recycling of workers exceeding `max_worker_rss` and of workers that crashed outright.
+The cost is worker start-up time (plus re-running `init_worker_code`) after each failure, which is why it is off by default: for a suite whose failures are self-contained it is pure overhead.
+
+### Retrying Failed Tests
+
+When several workers compete for a limited resource — GPU memory, RAM, a device that only allows so many contexts — a failure can mean "lost the race for the resource" rather than "the code is broken".
+Such a test typically passes when run on its own.
+
+The `retries` keyword argument re-runs tests that did not pass, up to `N` times, after the main run has completed:
+
+```julia
+runtests(MyPackage, ARGS; retries=1)
+```
+
+The retry environment is deliberately quiesced: all parallel workers have been stopped by then, and the retried tests run **sequentially on a single fresh worker**, so a test that failed only because of concurrent resource pressure gets an otherwise-idle system.
+If a test fails again, its worker is stopped before the next retry, so one failure cannot contaminate the following one.
+
+Only the final attempt of each test is recorded in the results, so a test that passes on retry is reported as passing and a persistently broken test is reported as failing exactly once.
+Retries are visible in the output, so flakiness is surfaced rather than hidden:
+
+```
+Retrying 2 failed test(s) on a fresh worker...
+  gpu/memory passed on retry
+  broken_test failed again
+```
+
+!!! note
+    Retries are skipped when the run was interrupted (e.g. `Ctrl+C`) or when `--quickfail` is
+    in effect, since in both cases the run stopped early on purpose.
+
+!!! tip
+    `recycle_on_failure` and `retries` address different halves of the same problem and work
+    well together: recycling keeps one bad test from cascading onto its worker during the run,
+    while retries give the tests that did fail a contention-free second chance.
+
 ## Custom Workers
 
 For tests that require specific environment variables or Julia flags, you can use the `test_worker` keyword argument to [`runtests`](@ref) to assign tests to custom workers:
@@ -303,3 +352,5 @@ function jltest {
 1. **Use custom workers sparingly**: Custom workers add overhead. Only use them when tests genuinely require different configurations.
 
 1. **Use `serial` for resource-intensive tests**: If a test allocates significant memory or uses exclusive hardware resources, mark it as serial rather than reducing `--jobs` globally. This keeps the rest of your suite running in parallel.
+
+1. **Don't paper over real failures with `retries`**: Retries are meant for failures caused by contention between concurrent workers, not for tests that are genuinely broken. Persistent failures still fail after their retries, and retried tests are reported as such, so keep an eye on which tests keep needing a second attempt.
diff --git a/docs/src/index.md b/docs/src/index.md
index cb89f41..41a948a 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -114,6 +114,15 @@ The `serial` keyword argument to [`runtests`](@ref) lets you designate specific
 for sequential execution, either before or after the parallel batch.
 See [Serial Tests](@ref) in the advanced usage guide for details.
 
+### Failure Recycling and Retries
+
+Workers are recycled when they crash or exceed the memory threshold.
+Additionally, `recycle_on_failure=true` recycles a worker after any failed test, so a test
+that corrupts process-wide state cannot poison later tests, and `retries=N` re-runs failed
+tests on an otherwise-idle system, to tell tests broken by resource contention apart from
+genuinely broken ones.
+See [Failure Handling](@ref) in the advanced usage guide for details.
+
 ### Real-time Progress
 
 The test runner provides real-time output showing:

From 821098be9ef1c6ce7d18d08a52b44dd37353a3b8 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Mon, 3 Aug 2026 11:22:11 -0500
Subject: [PATCH 03/21] Add tests for recycle_on_failure and retries

For `recycle_on_failure`, run a fixed sequence of failing and passing tests
with a single job and count the workers created: the default reuses one
worker for all of them, while `recycle_on_failure=true` needs a fresh worker
after each failure.

For `retries`, use a test that fails on its first attempt and passes on any
subsequent one (recording attempts in a file, since each attempt runs in a
different process) to check that a test rescued by a retry is reported as
passing, and that it is the only worker alive while it runs. A persistently
failing test is checked to exhaust its retries and still be reported exactly
once. Also cover that retries are off by default and skipped under
`--quickfail`.
---
 test/runtests.jl | 197 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 197 insertions(+)

diff --git a/test/runtests.jl b/test/runtests.jl
index 1cb7314..be1ca53 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1463,6 +1463,203 @@ end
     end
 end
 
+@testset "recycle_on_failure" begin
+    # Call `_runtests` throughout, so that we can enforce a run order, and use a single job,
+    # so that all tests share the same pool slot: a test only gets a new worker if the
+    # previous one was recycled.
+    testsuite = Dict(
+        "fail1" => :( @test false ),
+        "pass1" => :( @test true ),
+        "fail2" => :( @test false ),
+        "pass2" => :( @test true ),
+    )
+    tests = ["fail1", "pass1", "fail2", "pass2"]
+
+    @testset "workers are reused across failures by default" begin
+        io = IOBuffer()
+        old_id_counter = ParallelTestRunner.ID_COUNTER[]
+        @test_throws Test.FallbackTestSetException begin
+            ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests,
+                historical_durations=Dict{String, Float64}(),
+                stdout=io,
+                stderr=io,
+            )
+        end
+        str = String(take!(io))
+        @test contains(str, "FAILURE")
+        # A failing test does not recycle its worker, so a single one runs all four tests.
+        @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + 1
+    end
+
+    @testset "worker is recycled after a failed test" begin
+        io = IOBuffer()
+        old_id_counter = ParallelTestRunner.ID_COUNTER[]
+        @test_throws Test.FallbackTestSetException begin
+            ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests,
+                historical_durations=Dict{String, Float64}(),
+                stdout=io,
+                stderr=io,
+                recycle_on_failure=true,
+            )
+        end
+        str = String(take!(io))
+        @test contains(str, "FAILURE")
+        # `fail1` and `fail2` recycle their worker, so `pass1` and `pass2` each need a fresh
+        # one: 1 initial worker + 2 replacements.
+        @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + 3
+    end
+end
+
+@testset "retries" begin
+    # A test that fails on its first attempt and passes on any subsequent one, by recording
+    # attempts in a file: the worker running the retry is a different process, so the marker
+    # has to live outside of it.
+    flaky_test(marker, body=:( @test true )) = quote
+        if isfile($marker)
+            $body
+        else
+            touch($marker)
+            @test false
+        end
+    end
+
+    @testset "failed test passing on retry is reported as passing" begin
+        mktempdir() do dir
+            testsuite = Dict(
+                "flaky" => flaky_test(joinpath(dir, "flaky")),
+                "passes" => :( @test true ),
+            )
+            io = IOBuffer()
+            @show_if_error io ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests=["flaky", "passes"],
+                historical_durations=Dict{String, Float64}(),
+                stdout=io,
+                stderr=io,
+                retries=1,
+            )
+            str = String(take!(io))
+            # Only the failed test is retried, and its retried result is the one reported.
+            @test contains(str, "Retrying 1 failed test(s)")
+            @test contains(str, "flaky passed on retry")
+            @test !contains(str, "passes passed on retry")
+            @test contains(str, "SUCCESS")
+            # Two results in total: the failed attempt of `flaky` was replaced by the
+            # retried one, rather than reported next to it.
+            @test contains(str, r"Overall +\| +2 +2 ")
+        end
+    end
+
+    @testset "persistent failure is retried and reported once" begin
+        testsuite = Dict(
+            "always_fails" => :( @test false ),
+            "passes" => :( @test true ),
+        )
+        io = IOBuffer()
+        @test_throws Test.FallbackTestSetException begin
+            ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests=["always_fails", "passes"],
+                historical_durations=Dict{String, Float64}(),
+                stdout=io,
+                stderr=io,
+                retries=2,
+            )
+        end
+        str = String(take!(io))
+        @test contains(str, "FAILURE")
+        # Both retry rounds run, and each of them fails again.
+        @test length(collect(eachmatch(r"always_fails failed again", str))) == 2
+        # Despite the three attempts, the test is reported exactly once, as a failure.
+        @test contains(str, r"always_fails +\| +1 +1 ")
+    end
+
+    @testset "retried test runs alone" begin
+        mktempdir() do dir
+            # On its retry, the flaky test checks it is the only worker left alive.
+            check_alone = quote
+                children = _count_child_pids($(getpid()))
+                if children >= 0
+                    @test children == 1
+                end
+            end
+            testsuite = Dict(
+                "flaky" => flaky_test(joinpath(dir, "flaky"), check_alone),
+                "pass1" => :( @test true ),
+                "pass2" => :( @test true ),
+                "pass3" => :( @test true ),
+            )
+            io = IOBuffer()
+            @show_if_error io ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=3"]);
+                testsuite,
+                tests=["flaky", "pass1", "pass2", "pass3"],
+                historical_durations=Dict{String, Float64}(),
+                init_code=:(include($(joinpath(@__DIR__, "utils.jl")))),
+                stdout=io,
+                stderr=io,
+                retries=1,
+            )
+            str = String(take!(io))
+            @test contains(str, "flaky passed on retry")
+            @test contains(str, "SUCCESS")
+        end
+    end
+
+    @testset "no retries by default" begin
+        mktempdir() do dir
+            testsuite = Dict("flaky" => flaky_test(joinpath(dir, "flaky")))
+            io = IOBuffer()
+            @test_throws Test.FallbackTestSetException begin
+                ParallelTestRunner._runtests(
+                    ParallelTestRunner, parse_args(["--jobs=1"]);
+                    testsuite,
+                    tests=["flaky"],
+                    historical_durations=Dict{String, Float64}(),
+                    stdout=io,
+                    stderr=io,
+                )
+            end
+            str = String(take!(io))
+            @test !contains(str, "Retrying")
+            @test contains(str, "FAILURE")
+        end
+    end
+
+    @testset "quickfail skips retries" begin
+        mktempdir() do dir
+            testsuite = Dict(
+                "flaky" => flaky_test(joinpath(dir, "flaky")),
+                "passes" => :( @test true ),
+            )
+            io = IOBuffer()
+            @test_throws Test.FallbackTestSetException begin
+                ParallelTestRunner._runtests(
+                    ParallelTestRunner, parse_args(["--quickfail", "--jobs=1"]);
+                    testsuite,
+                    tests=["flaky", "passes"],
+                    historical_durations=Dict{String, Float64}(),
+                    stdout=io,
+                    stderr=io,
+                    retries=1,
+                )
+            end
+            str = String(take!(io))
+            # The run stopped early on purpose, retrying would defeat that.
+            @test !contains(str, "Retrying")
+            @test contains(str, "FAILURE")
+        end
+    end
+end
+
 # This testset should always be the last one, don't add anything after this.
 # We want to make sure there are no running workers at the end of the tests.
 @testset "no workers running" begin

From 2327a4565174b4680bcdf44a866eff7d5e577233 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Fri, 7 Aug 2026 10:34:03 -0500
Subject: [PATCH 04/21] Respect the test_worker hook when retrying failed tests

---
 src/ParallelTestRunner.jl | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 1a39c9c..f93ef21 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1554,14 +1554,27 @@ function _runtests(mod::Module, args::ParsedArgs;
                         "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
                         color = :yellow)
             for test in retryable
-                if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
-                    retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
-                                           exeflags, env)
+                # pass in init_worker_code to custom worker function if defined
+                wrkr = if init_worker_code == :()
+                    test_worker(test)
+                else
+                    test_worker(test, init_worker_code)
+                end
+                if wrkr !== nothing && !Malt.isrunning(wrkr)
+                    wrkr = nothing
+                end
+                custom = wrkr !== nothing
+                if !custom
+                    if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
+                        retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
+                                               exeflags, env)
+                    end
+                    wrkr = retry_wrkr
                 end
                 test_t0 = time()
                 result = try
-                    Malt.remote_eval_wait(Main, retry_wrkr.w, :(import ParallelTestRunner))
-                    Malt.remote_call_fetch(invokelatest, retry_wrkr.w, runtest,
+                    Malt.remote_eval_wait(Main, wrkr.w, :(import ParallelTestRunner))
+                    Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
                                            RecordType, testsuite[test], test,
                                            init_code, test_t0, custom_args)
                 catch ex
@@ -1569,16 +1582,22 @@ function _runtests(mod::Module, args::ParsedArgs;
                     ex
                 end
                 test_t1 = time()
-                output = @lock retry_wrkr.io String(take!(retry_wrkr.io[]))
+                output = @lock wrkr.io String(take!(wrkr.io[]))
                 filter!(r -> r.test != test, results.value)
                 push!(results.value, (; test, result, output, test_t0, test_t1))
                 if result isa AbstractTestRecord && !anynonpass(result[])
                     printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
                 else
                     printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
-                    # don't let a failure contaminate the next retry
-                    Malt.stop(retry_wrkr)
-                    retry_wrkr = nothing
+                    if !custom
+                        # don't let a failure contaminate the next retry
+                        Malt.stop(retry_wrkr)
+                        retry_wrkr = nothing
+                    end
+                end
+                # get rid of the custom worker
+                if custom && Malt.isrunning(wrkr)
+                    Malt.stop(wrkr)
                 end
             end
         end

From a0db67596c90d00f178d7b07ed7ad9faa43b2f70 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Fri, 7 Aug 2026 10:34:11 -0500
Subject: [PATCH 05/21] Remove redundant ParallelTestRunner import in the retry
 loop

---
 src/ParallelTestRunner.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index f93ef21..a3e53e8 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1573,7 +1573,6 @@ function _runtests(mod::Module, args::ParsedArgs;
                 end
                 test_t0 = time()
                 result = try
-                    Malt.remote_eval_wait(Main, wrkr.w, :(import ParallelTestRunner))
                     Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
                                            RecordType, testsuite[test], test,
                                            init_code, test_t0, custom_args)

From 20f2e863b971ba2a6253c4e2f447e2d6a3daeed9 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Fri, 7 Aug 2026 10:34:27 -0500
Subject: [PATCH 06/21] Document recycle_on_failure and retries in the runtests
 signature

---
 src/ParallelTestRunner.jl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index a3e53e8..9706356 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -871,7 +871,9 @@ end
              stderr = Base.stderr,
              max_worker_rss = get_max_worker_rss(),
              serial = String[],
-             serial_position::Symbol = :before)
+             serial_position::Symbol = :before,
+             recycle_on_failure::Bool = false,
+             retries::Integer = 0)
     runtests(mod::Module, ARGS; ...)
 
 Run Julia tests in parallel across multiple worker processes.
@@ -919,6 +921,10 @@ Several keyword arguments are also supported:
   testsuite; names that are valid but deselected by command-line filtering are ignored.
 - `serial_position`: When to run serial tests relative to the parallel batch.
   Must be `:before` (default) or `:after`.
+- `recycle_on_failure`: Whether to recycle a worker after any test that did not pass
+  (default: `false`). See the Failure Handling section below.
+- `retries`: How many times to re-run tests that did not pass after the main run completes
+  (default: `0`). See the Failure Handling section below.
 
 ## Command Line Options
 

From 56ae0454a4cc383d46e22d426e85585cb58a7ff7 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 17:04:06 -0300
Subject: [PATCH 07/21] Remove no longer necessary arguments

---
 test/runtests.jl | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index be1ca53..c8cf204 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1483,7 +1483,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=1"]);
                 testsuite,
                 tests,
-                historical_durations=Dict{String, Float64}(),
                 stdout=io,
                 stderr=io,
             )
@@ -1502,7 +1501,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=1"]);
                 testsuite,
                 tests,
-                historical_durations=Dict{String, Float64}(),
                 stdout=io,
                 stderr=io,
                 recycle_on_failure=true,
@@ -1540,7 +1538,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=1"]);
                 testsuite,
                 tests=["flaky", "passes"],
-                historical_durations=Dict{String, Float64}(),
                 stdout=io,
                 stderr=io,
                 retries=1,
@@ -1568,7 +1565,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=1"]);
                 testsuite,
                 tests=["always_fails", "passes"],
-                historical_durations=Dict{String, Float64}(),
                 stdout=io,
                 stderr=io,
                 retries=2,
@@ -1602,7 +1598,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=3"]);
                 testsuite,
                 tests=["flaky", "pass1", "pass2", "pass3"],
-                historical_durations=Dict{String, Float64}(),
                 init_code=:(include($(joinpath(@__DIR__, "utils.jl")))),
                 stdout=io,
                 stderr=io,
@@ -1623,7 +1618,6 @@ end
                     ParallelTestRunner, parse_args(["--jobs=1"]);
                     testsuite,
                     tests=["flaky"],
-                    historical_durations=Dict{String, Float64}(),
                     stdout=io,
                     stderr=io,
                 )
@@ -1646,7 +1640,6 @@ end
                     ParallelTestRunner, parse_args(["--quickfail", "--jobs=1"]);
                     testsuite,
                     tests=["flaky", "passes"],
-                    historical_durations=Dict{String, Float64}(),
                     stdout=io,
                     stderr=io,
                     retries=1,

From 09f101afe1de5e4ca083b5e9b857bc7851ad9466 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 17:21:27 -0300
Subject: [PATCH 08/21] Use existing test phase machinery to implement retries

---
 src/ParallelTestRunner.jl | 146 +++++++++++++++++++++-----------------
 1 file changed, 80 insertions(+), 66 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 9706356..3259dee 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1505,6 +1505,20 @@ function _runtests(mod::Module, args::ParsedArgs;
                 end
             end
         end
+
+        # retries
+        for i in 1:retries
+            retry_tests = [r.test for r in results.value
+                                if r.result isa Exception || anynonpass(r.result[])]
+            isempty(retry_tests) && break
+
+            println(io_ctx.stdout, "Retrying $(length(retry_tests)) failed tests ($i)")
+            sem = Base.Semaphore(1)
+            shared_worker = serial_worker
+            filter!(r -> r.test ∉ retry_tests, results.value)
+
+            run_test_phase(retry_tests, sem, shared_worker)
+        end
     catch err
         interrupted = true
         if !(err isa InterruptException)
@@ -1544,72 +1558,72 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
     end
 
-    # retry failed tests, if requested: sequentially, on a single fresh worker, with every
-    # other worker gone — tests that failed due to resource pressure (e.g. GPU memory
-    # oversubscription from concurrent workers) reliably pass on an otherwise-idle system.
-    # only the retried result is reported; persistent failures fail again and are reported
-    # exactly once.
-    if retries > 0 && !interrupted && args.quickfail === nothing
-        local retry_wrkr = nothing
-        for round in 1:retries
-            retryable = [r.test for r in results.value
-                         if r.result isa Exception || anynonpass(r.result[])]
-            isempty(retryable) && break
-            println(io_ctx.stdout)
-            printstyled(io_ctx.stdout,
-                        "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
-                        color = :yellow)
-            for test in retryable
-                # pass in init_worker_code to custom worker function if defined
-                wrkr = if init_worker_code == :()
-                    test_worker(test)
-                else
-                    test_worker(test, init_worker_code)
-                end
-                if wrkr !== nothing && !Malt.isrunning(wrkr)
-                    wrkr = nothing
-                end
-                custom = wrkr !== nothing
-                if !custom
-                    if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
-                        retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
-                                               exeflags, env)
-                    end
-                    wrkr = retry_wrkr
-                end
-                test_t0 = time()
-                result = try
-                    Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
-                                           RecordType, testsuite[test], test,
-                                           init_code, test_t0, custom_args)
-                catch ex
-                    isa(ex, InterruptException) && rethrow()
-                    ex
-                end
-                test_t1 = time()
-                output = @lock wrkr.io String(take!(wrkr.io[]))
-                filter!(r -> r.test != test, results.value)
-                push!(results.value, (; test, result, output, test_t0, test_t1))
-                if result isa AbstractTestRecord && !anynonpass(result[])
-                    printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
-                else
-                    printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
-                    if !custom
-                        # don't let a failure contaminate the next retry
-                        Malt.stop(retry_wrkr)
-                        retry_wrkr = nothing
-                    end
-                end
-                # get rid of the custom worker
-                if custom && Malt.isrunning(wrkr)
-                    Malt.stop(wrkr)
-                end
-            end
-        end
-        if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr)
-            Malt.stop(retry_wrkr)
-        end
-    end
+    # # retry failed tests, if requested: sequentially, on a single fresh worker, with every
+    # # other worker gone — tests that failed due to resource pressure (e.g. GPU memory
+    # # oversubscription from concurrent workers) reliably pass on an otherwise-idle system.
+    # # only the retried result is reported; persistent failures fail again and are reported
+    # # exactly once.
+    # if retries > 0 && !interrupted && args.quickfail === nothing
+    #     local retry_wrkr = nothing
+    #     for round in 1:retries
+    #         retryable = [r.test for r in results.value
+    #                      if r.result isa Exception || anynonpass(r.result[])]
+    #         isempty(retryable) && break
+    #         println(io_ctx.stdout)
+    #         printstyled(io_ctx.stdout,
+    #                     "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
+    #                     color = :yellow)
+    #         for test in retryable
+    #             # pass in init_worker_code to custom worker function if defined
+    #             wrkr = if init_worker_code == :()
+    #                 test_worker(test)
+    #             else
+    #                 test_worker(test, init_worker_code)
+    #             end
+    #             if wrkr !== nothing && !Malt.isrunning(wrkr)
+    #                 wrkr = nothing
+    #             end
+    #             custom = wrkr !== nothing
+    #             if !custom
+    #                 if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
+    #                     retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
+    #                                            exeflags, env)
+    #                 end
+    #                 wrkr = retry_wrkr
+    #             end
+    #             test_t0 = time()
+    #             result = try
+    #                 Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
+    #                                        RecordType, testsuite[test], test,
+    #                                        init_code, test_t0, custom_args)
+    #             catch ex
+    #                 isa(ex, InterruptException) && rethrow()
+    #                 ex
+    #             end
+    #             test_t1 = time()
+    #             output = @lock wrkr.io String(take!(wrkr.io[]))
+    #             filter!(r -> r.test != test, results.value)
+    #             push!(results.value, (; test, result, output, test_t0, test_t1))
+    #             if result isa AbstractTestRecord && !anynonpass(result[])
+    #                 printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
+    #             else
+    #                 printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
+    #                 if !custom
+    #                     # don't let a failure contaminate the next retry
+    #                     Malt.stop(retry_wrkr)
+    #                     retry_wrkr = nothing
+    #                 end
+    #             end
+    #             # get rid of the custom worker
+    #             if custom && Malt.isrunning(wrkr)
+    #                 Malt.stop(wrkr)
+    #             end
+    #         end
+    #     end
+    #     if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr)
+    #         Malt.stop(retry_wrkr)
+    #     end
+    # end
 
     # print the output generated by each testset
     # (`@sync` above joined all writers, so `results` is quiescent from here on)

From bd771594645847cc8ebe171577cac98d76a788af Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 17:29:16 -0300
Subject: [PATCH 09/21] Print failures of non-final attempts yellow

---
 src/ParallelTestRunner.jl | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 3259dee..e2b00f9 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -168,6 +168,7 @@ struct TestIOContext
     alloc_align::Int
     rss_align::Int
     max_worker_rss::Int
+    nonpass_face::Ref{Symbol}
 end
 
 function test_IOContext(::Type{<:AbstractTestRecord}, stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int, verbose::Bool, max_worker_rss::Int)
@@ -182,7 +183,7 @@ function test_IOContext(::Type{<:AbstractTestRecord}, stdout::IO, stderr::IO, lo
 
     return TestIOContext(
         stdout, stderr, color, verbose, lock, name_align, elapsed_align, compile_align, gc_align, percent_align,
-        alloc_align, rss_align, max_worker_rss
+        alloc_align, rss_align, max_worker_rss, Ref(:ptr_error)
     )
 end
 
@@ -292,7 +293,7 @@ function print_test_failed(record::AbstractTestRecord, wrkr, test, ctx::TestIOCo
 
         # TODO: print other stats?
 
-        out_str = styled"{ptr_error:$test$padded_wrkr │$padded_time │$padded_init_time$failed_str}\n"
+        out_str = styled"{$(ctx.nonpass_face[]):$test$padded_wrkr │$padded_time │$padded_init_time$failed_str}\n"
         print(ctx.stderr, out_str)
         flush(ctx.stderr)
     finally
@@ -304,7 +305,7 @@ function print_test_crashed(::Type{<:AbstractTestRecord}, wrkr, test, ctx::TestI
     lock(ctx.lock)
     try
         padded_wrkr = lpad("($wrkr)", ctx.name_align - textwidth(test) + 1, " ")
-        out_str = styled"{ptr_error:$(test)$padded_wrkr │$(\" \"^ctx.elapsed_align) crashed at $(now())}\n"
+        out_str = styled"{$(ctx.nonpass_face[]):$(test)$padded_wrkr │$(\" \"^ctx.elapsed_align) crashed at $(now())}\n"
         print(ctx.stderr, out_str)
         flush(ctx.stderr)
     finally
@@ -1490,6 +1491,7 @@ function _runtests(mod::Module, args::ParsedArgs;
     end
     try
         phases = test_phases
+        retries > 0 && (io_ctx.nonpass_face[] = :ptr_warn)
         for i in 1:length(phases)
             phase_tests, sem, shared_worker = phases[i]
             isempty(phase_tests) && continue
@@ -1508,11 +1510,12 @@ function _runtests(mod::Module, args::ParsedArgs;
 
         # retries
         for i in 1:retries
+            retries == i && (io_ctx.nonpass_face[] = :ptr_error)
             retry_tests = [r.test for r in results.value
                                 if r.result isa Exception || anynonpass(r.result[])]
             isempty(retry_tests) && break
 
-            println(io_ctx.stdout, "Retrying $(length(retry_tests)) failed tests ($i)")
+            println(io_ctx.stdout, styled"{ptr_default:Retrying $(length(retry_tests)) failed tests ($i)}")
             sem = Base.Semaphore(1)
             shared_worker = serial_worker
             filter!(r -> r.test ∉ retry_tests, results.value)

From 7d8c6caf9b98422f399a77f811be7d769cc34627 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 17:54:56 -0300
Subject: [PATCH 10/21] Print properly

---
 src/ParallelTestRunner.jl | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index e2b00f9..b8f78a8 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1302,6 +1302,18 @@ function _runtests(mod::Module, args::ParsedArgs;
 
                         clear_status()
                         print_test_crashed(RecordType, wrkr, test_name, io_ctx)
+
+                    elseif msg_type === :retry
+                        tests_n, retry_n = msg[2], msg[3]
+
+                        clear_status()
+                        lock(io_ctx.lock)
+                        try
+                            println(io_ctx.stdout, styled"{ptr_default:Retrying $tests_n failed test$(tests_n > 1 ? \"s\" : \" \") ($retry_n)}")
+                            flush(io_ctx.stdout)
+                        finally
+                            unlock(io_ctx.lock)
+                        end
                     end
                 end
 
@@ -1515,7 +1527,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                                 if r.result isa Exception || anynonpass(r.result[])]
             isempty(retry_tests) && break
 
-            println(io_ctx.stdout, styled"{ptr_default:Retrying $(length(retry_tests)) failed tests ($i)}")
+            put!(printer_channel, (:retry, length(retry_tests), i))
             sem = Base.Semaphore(1)
             shared_worker = serial_worker
             filter!(r -> r.test ∉ retry_tests, results.value)

From f60373877a21bb0076aef3b1753b2d716dc83d94 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 18:00:32 -0300
Subject: [PATCH 11/21] Fix most tests

---
 test/runtests.jl | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index c8cf204..ee7062a 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1544,9 +1544,7 @@ end
             )
             str = String(take!(io))
             # Only the failed test is retried, and its retried result is the one reported.
-            @test contains(str, "Retrying 1 failed test(s)")
-            @test contains(str, "flaky passed on retry")
-            @test !contains(str, "passes passed on retry")
+            @test contains(str, "Retrying 1 failed test")
             @test contains(str, "SUCCESS")
             # Two results in total: the failed attempt of `flaky` was replaced by the
             # retried one, rather than reported next to it.
@@ -1573,7 +1571,7 @@ end
         str = String(take!(io))
         @test contains(str, "FAILURE")
         # Both retry rounds run, and each of them fails again.
-        @test length(collect(eachmatch(r"always_fails failed again", str))) == 2
+        @test length(collect(eachmatch(r"always_fails.*failed", str))) == 3
         # Despite the three attempts, the test is reported exactly once, as a failure.
         @test contains(str, r"always_fails +\| +1 +1 ")
     end
@@ -1604,7 +1602,7 @@ end
                 retries=1,
             )
             str = String(take!(io))
-            @test contains(str, "flaky passed on retry")
+            @test length(collect(eachmatch(r"failed", str))) == 2
             @test contains(str, "SUCCESS")
         end
     end

From 116706b7ced6aba3e4bc3c6c76be380ade587c89 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sat, 8 Aug 2026 14:54:20 -0300
Subject: [PATCH 12/21] Remove old code

---
 src/ParallelTestRunner.jl | 67 ---------------------------------------
 1 file changed, 67 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index b8f78a8..0fecbc9 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1573,73 +1573,6 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
     end
 
-    # # retry failed tests, if requested: sequentially, on a single fresh worker, with every
-    # # other worker gone — tests that failed due to resource pressure (e.g. GPU memory
-    # # oversubscription from concurrent workers) reliably pass on an otherwise-idle system.
-    # # only the retried result is reported; persistent failures fail again and are reported
-    # # exactly once.
-    # if retries > 0 && !interrupted && args.quickfail === nothing
-    #     local retry_wrkr = nothing
-    #     for round in 1:retries
-    #         retryable = [r.test for r in results.value
-    #                      if r.result isa Exception || anynonpass(r.result[])]
-    #         isempty(retryable) && break
-    #         println(io_ctx.stdout)
-    #         printstyled(io_ctx.stdout,
-    #                     "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
-    #                     color = :yellow)
-    #         for test in retryable
-    #             # pass in init_worker_code to custom worker function if defined
-    #             wrkr = if init_worker_code == :()
-    #                 test_worker(test)
-    #             else
-    #                 test_worker(test, init_worker_code)
-    #             end
-    #             if wrkr !== nothing && !Malt.isrunning(wrkr)
-    #                 wrkr = nothing
-    #             end
-    #             custom = wrkr !== nothing
-    #             if !custom
-    #                 if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
-    #                     retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
-    #                                            exeflags, env)
-    #                 end
-    #                 wrkr = retry_wrkr
-    #             end
-    #             test_t0 = time()
-    #             result = try
-    #                 Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
-    #                                        RecordType, testsuite[test], test,
-    #                                        init_code, test_t0, custom_args)
-    #             catch ex
-    #                 isa(ex, InterruptException) && rethrow()
-    #                 ex
-    #             end
-    #             test_t1 = time()
-    #             output = @lock wrkr.io String(take!(wrkr.io[]))
-    #             filter!(r -> r.test != test, results.value)
-    #             push!(results.value, (; test, result, output, test_t0, test_t1))
-    #             if result isa AbstractTestRecord && !anynonpass(result[])
-    #                 printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
-    #             else
-    #                 printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
-    #                 if !custom
-    #                     # don't let a failure contaminate the next retry
-    #                     Malt.stop(retry_wrkr)
-    #                     retry_wrkr = nothing
-    #                 end
-    #             end
-    #             # get rid of the custom worker
-    #             if custom && Malt.isrunning(wrkr)
-    #                 Malt.stop(wrkr)
-    #             end
-    #         end
-    #     end
-    #     if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr)
-    #         Malt.stop(retry_wrkr)
-    #     end
-    # end
-
     # print the output generated by each testset
     # (`@sync` above joined all writers, so `results` is quiescent from here on)
     for (testname, result, output, _start, _stop) in results.value

From 7d462d8ad6c62eb7c96812738999122306717cf3 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sat, 8 Aug 2026 15:03:39 -0300
Subject: [PATCH 13/21] Fix test

---
 src/ParallelTestRunner.jl | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 0fecbc9..97b346d 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1521,18 +1521,20 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
 
         # retries
-        for i in 1:retries
-            retries == i && (io_ctx.nonpass_face[] = :ptr_error)
-            retry_tests = [r.test for r in results.value
-                                if r.result isa Exception || anynonpass(r.result[])]
-            isempty(retry_tests) && break
-
-            put!(printer_channel, (:retry, length(retry_tests), i))
-            sem = Base.Semaphore(1)
-            shared_worker = serial_worker
-            filter!(r -> r.test ∉ retry_tests, results.value)
-
-            run_test_phase(retry_tests, sem, shared_worker)
+        if retries > 0 && !interrupted && args.quickfail === nothing
+            for i in 1:retries
+                retries == i && (io_ctx.nonpass_face[] = :ptr_error)
+                retry_tests = [r.test for r in results.value
+                                    if r.result isa Exception || anynonpass(r.result[])]
+                isempty(retry_tests) && break
+
+                put!(printer_channel, (:retry, length(retry_tests), i))
+                sem = Base.Semaphore(1)
+                shared_worker = serial_worker
+                filter!(r -> r.test ∉ retry_tests, results.value)
+
+                run_test_phase(retry_tests, sem, shared_worker)
+            end
         end
     catch err
         interrupted = true

From 95cad6dac93410641cac4c3f449ce6ad05f77b2d Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sun, 9 Aug 2026 20:09:27 -0300
Subject: [PATCH 14/21] Tweak documentation

---
 docs/src/advanced.md      | 14 ++++++--------
 docs/src/index.md         |  9 +++++----
 src/ParallelTestRunner.jl |  6 ++----
 3 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/docs/src/advanced.md b/docs/src/advanced.md
index b749b7d..e49acab 100644
--- a/docs/src/advanced.md
+++ b/docs/src/advanced.md
@@ -189,11 +189,10 @@ runtests(MyPackage, ARGS; recycle_on_failure=true)
 ```
 
 This complements the existing recycling of workers exceeding `max_worker_rss` and of workers that crashed outright.
-The cost is worker start-up time (plus re-running `init_worker_code`) after each failure, which is why it is off by default: for a suite whose failures are self-contained it is pure overhead.
 
 ### Retrying Failed Tests
 
-When several workers compete for a limited resource — GPU memory, RAM, a device that only allows so many contexts — a failure can mean "lost the race for the resource" rather than "the code is broken".
+When several workers compete for a limited resource (usually memory), a failure can mean "lost the race for the resource" rather than "the code is broken".
 Such a test typically passes when run on its own.
 
 The `retries` keyword argument re-runs tests that did not pass, up to `N` times, after the main run has completed:
@@ -202,16 +201,15 @@ The `retries` keyword argument re-runs tests that did not pass, up to `N` times,
 runtests(MyPackage, ARGS; retries=1)
 ```
 
-The retry environment is deliberately quiesced: all parallel workers have been stopped by then, and the retried tests run **sequentially on a single fresh worker**, so a test that failed only because of concurrent resource pressure gets an otherwise-idle system.
+Retried tests run **sequentially on a single fresh worker**, so a test that failed only because of concurrent resource pressure gets an otherwise-idle system.
 If a test fails again, its worker is stopped before the next retry, so one failure cannot contaminate the following one.
 
-Only the final attempt of each test is recorded in the results, so a test that passes on retry is reported as passing and a persistently broken test is reported as failing exactly once.
+Only the final attempt of each test is recorded in the results, so a test that passes on retry is reported as passing and a persistently broken test is reported as failing.
 Retries are visible in the output, so flakiness is surfaced rather than hidden:
 
 ```
-Retrying 2 failed test(s) on a fresh worker...
-  gpu/memory passed on retry
-  broken_test failed again
+Retrying 1 failed test  (1)
+fails      (8) │     0.05 │   failed at 2026-08-08T15:10:15.526
 ```
 
 !!! note
@@ -353,4 +351,4 @@ function jltest {
 
 1. **Use `serial` for resource-intensive tests**: If a test allocates significant memory or uses exclusive hardware resources, mark it as serial rather than reducing `--jobs` globally. This keeps the rest of your suite running in parallel.
 
-1. **Don't paper over real failures with `retries`**: Retries are meant for failures caused by contention between concurrent workers, not for tests that are genuinely broken. Persistent failures still fail after their retries, and retried tests are reported as such, so keep an eye on which tests keep needing a second attempt.
+1. **Only use `retries` for worker contention-related failures**: Not all intermittent failures are caused by parallel worker resource contention. Ensure you aren't masking real test failures when using this feature.
diff --git a/docs/src/index.md b/docs/src/index.md
index 41a948a..b7d569c 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -117,10 +117,11 @@ See [Serial Tests](@ref) in the advanced usage guide for details.
 ### Failure Recycling and Retries
 
 Workers are recycled when they crash or exceed the memory threshold.
-Additionally, `recycle_on_failure=true` recycles a worker after any failed test, so a test
-that corrupts process-wide state cannot poison later tests, and `retries=N` re-runs failed
-tests on an otherwise-idle system, to tell tests broken by resource contention apart from
-genuinely broken ones.
+Additionally, [`runtests`](@ref) has two keyword arguments to further customize
+failure hanlding. Setting `recycle_on_failure=true` recycles a worker after any
+failed test, so a test that corrupts process-wide state cannot poison later tests,
+and `retries=N` re-runs failed tests sequentially up to `N` times to reduce false
+failures caused by resource contention.
 See [Failure Handling](@ref) in the advanced usage guide for details.
 
 ### Real-time Progress
diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 97b346d..83df034 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1012,10 +1012,8 @@ With `recycle_on_failure = true`, a worker is recycled after any test that did n
 a test that corrupts process-wide state (e.g. wedges a GPU driver) cannot poison subsequent
 tests on the same worker.
 
-With `retries = N` (default 0), tests that did not pass are re-run up to `N` times after
-the main run completes — sequentially, on a single fresh worker, with all other workers
-stopped — so tests that failed due to resource pressure from concurrent workers get an
-otherwise-idle system. Only the final attempt of each test is reported.
+With `retries = N` (default 0), tests that did not pass are re-run sequantially up to `N`
+times after the main run completes. Only the final attempt of each test is reported.
 """
 function runtests(mod::Module, args::ParsedArgs;
                   testsuite::Dict{String,Expr} = find_tests(pwd()),

From a07a9925e336867a9659de2fff5982954da76895 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sun, 9 Aug 2026 20:14:38 -0300
Subject: [PATCH 15/21] Failed tests print red when --quickfail is set

---
 src/ParallelTestRunner.jl | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 83df034..858ddcd 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1501,7 +1501,10 @@ function _runtests(mod::Module, args::ParsedArgs;
     end
     try
         phases = test_phases
-        retries > 0 && (io_ctx.nonpass_face[] = :ptr_warn)
+
+        potential_retries = retries > 0 && !interrupted && args.quickfail === nothing
+
+        potential_retries && (io_ctx.nonpass_face[] = :ptr_warn)
         for i in 1:length(phases)
             phase_tests, sem, shared_worker = phases[i]
             isempty(phase_tests) && continue
@@ -1519,7 +1522,7 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
 
         # retries
-        if retries > 0 && !interrupted && args.quickfail === nothing
+        if potential_retries
             for i in 1:retries
                 retries == i && (io_ctx.nonpass_face[] = :ptr_error)
                 retry_tests = [r.test for r in results.value

From b2a6b2ac01274b9491fdb72e5abe4dc0c6073f96 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sun, 9 Aug 2026 20:44:43 -0300
Subject: [PATCH 16/21] Typo

---
 src/ParallelTestRunner.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 858ddcd..7b56778 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1012,7 +1012,7 @@ With `recycle_on_failure = true`, a worker is recycled after any test that did n
 a test that corrupts process-wide state (e.g. wedges a GPU driver) cannot poison subsequent
 tests on the same worker.
 
-With `retries = N` (default 0), tests that did not pass are re-run sequantially up to `N`
+With `retries = N` (default 0), tests that did not pass are re-run sequentially up to `N`
 times after the main run completes. Only the final attempt of each test is reported.
 """
 function runtests(mod::Module, args::ParsedArgs;

From b7d023f2ec5e247802c3895df25d665ad41c214e Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 10 Aug 2026 13:50:57 -0300
Subject: [PATCH 17/21] Document failure printing colour

---
 docs/src/advanced.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/src/advanced.md b/docs/src/advanced.md
index e49acab..6da9bab 100644
--- a/docs/src/advanced.md
+++ b/docs/src/advanced.md
@@ -212,6 +212,10 @@ Retrying 1 failed test  (1)
 fails      (8) │     0.05 │   failed at 2026-08-08T15:10:15.526
 ```
 
+While a test still has an attempt left, its failure is printed in yellow, and the final
+attempt is printed in red. A red line therefore always marks the result that will be reported,
+and a yellow one marks a result that may still be replaced.
+
 !!! note
     Retries are skipped when the run was interrupted (e.g. `Ctrl+C`) or when `--quickfail` is
     in effect, since in both cases the run stopped early on purpose.

From 154cf4555a7dedfd4333236de0c7fe7e6b3ec7ea Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 10 Aug 2026 13:58:37 -0300
Subject: [PATCH 18/21] Address "Single fresh worker is no longer guaranteed"

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/ParallelTestRunner.jl | 32 +++++++++++++++++++-------------
 test/runtests.jl          | 35 ++++++++++++++++++++++++++++++++++-
 2 files changed, 53 insertions(+), 14 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 7b56778..2feeb98 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1349,9 +1349,9 @@ function _runtests(mod::Module, args::ParsedArgs;
 
     tests_to_start = Threads.Atomic{Int}(length(tests))
     interrupted = false
-    # After parallel-before-serial: stop extra workers so only one process is alive for
-    # serial tests, but keep one parallel worker so we do not add a third addworker (ID_COUNTER).
-    function drain_pool_leaving_one_worker!(pool, njobs)
+    # Stop every all-but-`n` workers in the pool.Only safe at a
+    # phase boundary, where all `njobs` slots have been returned.
+    function drain_pool_leaving_n_workers!(pool, njobs, n)
         alive = PTRWorker[]
         for _ in 1:njobs
             p = take!(pool)
@@ -1359,18 +1359,17 @@ function _runtests(mod::Module, args::ParsedArgs;
                 push!(alive, p)
             end
         end
-        while length(alive) > 1
+        while length(alive) > n
             Malt.stop(pop!(alive))
         end
-        kept = isempty(alive) ? nothing : alive[1]
-        if kept !== nothing
-            put!(pool, kept)
+        for p in alive
+            put!(pool, p)
         end
-        for _ in 1:(njobs - (kept === nothing ? 0 : 1))
+        for _ in 1:(njobs - length(alive))
             put!(pool, nothing)
         end
     end
-    function run_test_phase(phase_tests, sem, shared_worker)
+    function run_test_phase(phase_tests, sem, shared_worker; force_recycle::Bool=false)
         # for serial phases, reserve one pool slot for the shared worker
         if !isnothing(shared_worker)
             shared_worker[] = take!(worker_pool)
@@ -1446,7 +1445,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                                   # the worker has reached the max-rss limit, recycle it
                                   # so future tests start with a smaller working set
                                   Malt.stop(wrkr)
-                              elseif recycle_on_failure && anynonpass(result[])
+                              elseif (recycle_on_failure || force_recycle) && anynonpass(result[])
                                   # a failing test may have left the worker in a bad state
                                   # (e.g. a wedged GPU driver whose every later allocation
                                   # fails); recycle it so future tests get a fresh process
@@ -1512,11 +1511,12 @@ function _runtests(mod::Module, args::ParsedArgs;
             run_test_phase(phase_tests, sem, shared_worker)
 
             # parallel workers are not stopped while serial tests remain (tests_to_start > 0);
-            # drain before serial-after so only one worker is alive for the serial phase
+            # drain before serial-after so only one worker is alive for the serial phase.
+            # one is kept rather than none so we do not add a third addworker (ID_COUNTER).
             if isnothing(shared_worker) && i < length(phases)
                 next_tests, _, next_sw = phases[i+1]
                 if !isempty(next_tests) && !isnothing(next_sw)
-                    drain_pool_leaving_one_worker!(worker_pool, jobs)
+                    drain_pool_leaving_n_workers!(worker_pool, jobs, 1)
                 end
             end
         end
@@ -1534,7 +1534,13 @@ function _runtests(mod::Module, args::ParsedArgs;
                 shared_worker = serial_worker
                 filter!(r -> r.test ∉ retry_tests, results.value)
 
-                run_test_phase(retry_tests, sem, shared_worker)
+                # retries run on an otherwise-idle system: stop every worker left over
+                # from the previous phase, so the retry worker is spawned fresh below and
+                # no sibling process competes with it. `force_recycle` keeps it that way
+                # after each test that does not pass.
+                drain_pool_leaving_n_workers!(worker_pool, jobs, 0)
+
+                run_test_phase(retry_tests, sem, shared_worker; force_recycle=true)
             end
         end
     catch err
diff --git a/test/runtests.jl b/test/runtests.jl
index ee7062a..b52d4cc 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1576,7 +1576,13 @@ end
         @test contains(str, r"always_fails +\| +1 +1 ")
     end
 
-    @testset "retried test runs alone" begin
+    # `serial_position=:after` returns the live serial worker to the pool immediately
+    # before the retry phase, so it is the configuration where the "alone" invariant is
+    # easiest to break.
+    @testset "retried test runs alone (serial=$serial, $serial_position)" for
+            (serial, serial_position) in ((String[], :before),
+                                          (["pass3"], :before),
+                                          (["pass3"], :after))
         mktempdir() do dir
             # On its retry, the flaky test checks it is the only worker left alive.
             check_alone = quote
@@ -1597,6 +1603,8 @@ end
                 testsuite,
                 tests=["flaky", "pass1", "pass2", "pass3"],
                 init_code=:(include($(joinpath(@__DIR__, "utils.jl")))),
+                serial,
+                serial_position,
                 stdout=io,
                 stderr=io,
                 retries=1,
@@ -1607,6 +1615,31 @@ end
         end
     end
 
+    @testset "failing retry does not reuse its worker" begin
+        testsuite = Dict(
+            "failA" => :( @test false ),
+            "failB" => :( @test false ),
+        )
+        io = IOBuffer()
+        @test_throws Test.FallbackTestSetException begin
+            ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests=["failA", "failB"],
+                stdout=io,
+                stderr=io,
+                retries=1,
+            )
+        end
+        str = String(take!(io))
+        main, retry = split(str, "Retrying")
+        ids(s) = [m[1] for m in eachmatch(r"fail[AB] +\((\d+)\)", s)]
+        # the main run reuses one worker: `recycle_on_failure` is off by default
+        @test length(ids(main)) == 2 && allequal(ids(main))
+        # the retry round recycles after every non-pass, so each test gets its own worker
+        @test length(ids(retry)) == 2 && allunique(ids(retry))
+    end
+
     @testset "no retries by default" begin
         mktempdir() do dir
             testsuite = Dict("flaky" => flaky_test(joinpath(dir, "flaky")))

From cb937bf4488a13a1d4cec6bf96006235f26961eb Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 10 Aug 2026 14:13:40 -0300
Subject: [PATCH 19/21] Remove unnecessary `interrupted`

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/ParallelTestRunner.jl | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 2feeb98..4e4f8ac 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1348,7 +1348,6 @@ function _runtests(mod::Module, args::ParsedArgs;
     #
 
     tests_to_start = Threads.Atomic{Int}(length(tests))
-    interrupted = false
     # Stop every all-but-`n` workers in the pool.Only safe at a
     # phase boundary, where all `njobs` slots have been returned.
     function drain_pool_leaving_n_workers!(pool, njobs, n)
@@ -1369,7 +1368,9 @@ function _runtests(mod::Module, args::ParsedArgs;
             put!(pool, nothing)
         end
     end
-    function run_test_phase(phase_tests, sem, shared_worker; force_recycle::Bool=false)
+    # `retry_mode` forces worker recycling after every test and enables
+    # deletion of an old failed run of the test that just finished
+    function run_test_phase(phase_tests, sem, shared_worker; retry_mode::Bool=false)
         # for serial phases, reserve one pool slot for the shared worker
         if !isnothing(shared_worker)
             shared_worker[] = take!(worker_pool)
@@ -1431,7 +1432,13 @@ function _runtests(mod::Module, args::ParsedArgs;
                           end
                           test_t1 = time()
                           output = @lock wrkr.io String(take!(wrkr.io[]))
-                          @lock results push!(results[], (; test, result, output, test_t0, test_t1))
+                          # a retry drops the record of the attempt it re-runs only once it
+                          # has one to put in its place: dropping them up front would lose
+                          # them outright if the phase is interrupted
+                          @lock results begin
+                              retry_mode && filter!(r -> r.test != test, results[])
+                              push!(results[], (; test, result, output, test_t0, test_t1))
+                          end
 
                           # act on the results
                           if result isa AbstractTestRecord
@@ -1445,7 +1452,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                                   # the worker has reached the max-rss limit, recycle it
                                   # so future tests start with a smaller working set
                                   Malt.stop(wrkr)
-                              elseif (recycle_on_failure || force_recycle) && anynonpass(result[])
+                              elseif (recycle_on_failure || retry_mode) && anynonpass(result[])
                                   # a failing test may have left the worker in a bad state
                                   # (e.g. a wedged GPU driver whose every later allocation
                                   # fails); recycle it so future tests get a fresh process
@@ -1501,7 +1508,7 @@ function _runtests(mod::Module, args::ParsedArgs;
     try
         phases = test_phases
 
-        potential_retries = retries > 0 && !interrupted && args.quickfail === nothing
+        potential_retries = retries > 0 && args.quickfail === nothing
 
         potential_retries && (io_ctx.nonpass_face[] = :ptr_warn)
         for i in 1:length(phases)
@@ -1524,6 +1531,11 @@ function _runtests(mod::Module, args::ParsedArgs;
         # retries
         if potential_retries
             for i in 1:retries
+                # `stop_work()` may have been called from a worker task or the printer
+                # without any exception reaching the `catch` below, so we cannot assume we
+                # got here normally; there is no point retrying a run being torn down.
+                done[] && break
+
                 retries == i && (io_ctx.nonpass_face[] = :ptr_error)
                 retry_tests = [r.test for r in results.value
                                     if r.result isa Exception || anynonpass(r.result[])]
@@ -1532,19 +1544,17 @@ function _runtests(mod::Module, args::ParsedArgs;
                 put!(printer_channel, (:retry, length(retry_tests), i))
                 sem = Base.Semaphore(1)
                 shared_worker = serial_worker
-                filter!(r -> r.test ∉ retry_tests, results.value)
 
                 # retries run on an otherwise-idle system: stop every worker left over
                 # from the previous phase, so the retry worker is spawned fresh below and
-                # no sibling process competes with it. `force_recycle` keeps it that way
+                # no sibling process competes with it. `retry_mode` keeps it that way
                 # after each test that does not pass.
                 drain_pool_leaving_n_workers!(worker_pool, jobs, 0)
 
-                run_test_phase(retry_tests, sem, shared_worker; force_recycle=true)
+                run_test_phase(retry_tests, sem, shared_worker; retry_mode=true)
             end
         end
     catch err
-        interrupted = true
         if !(err isa InterruptException)
             println(io_ctx.stderr, "\nCaught an error, stopping...")
         end

From 905cc687b46d8341c163d9ce772e7b40e2758b9b Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 10 Aug 2026 14:45:35 -0300
Subject: [PATCH 20/21] Fix potential race in nonpass print colour

---
 src/ParallelTestRunner.jl | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 4e4f8ac..37a7791 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1263,6 +1263,8 @@ function _runtests(mod::Module, args::ParsedArgs;
     # (:started, test_name, worker_id)
     # (:finished, test_name, worker_id, record)
     # (:crashed, test_name, worker_id, test_time)
+    # (:retry, tests_n, retry_n)
+    # (:nonpass_color, color)
     printer_channel = Channel{Tuple}(100)
 
     printer_task = @async begin
@@ -1312,6 +1314,12 @@ function _runtests(mod::Module, args::ParsedArgs;
                         finally
                             unlock(io_ctx.lock)
                         end
+
+                    elseif msg_type === :nonpass_face
+                        # routed through the channel rather than set directly so it lands
+                        # in order with the results it applies to: the coordinator flips it
+                        # while this task may still be draining the previous round
+                        io_ctx.nonpass_face[] = msg[2]
                     end
                 end
 
@@ -1510,7 +1518,7 @@ function _runtests(mod::Module, args::ParsedArgs;
 
         potential_retries = retries > 0 && args.quickfail === nothing
 
-        potential_retries && (io_ctx.nonpass_face[] = :ptr_warn)
+        potential_retries && put!(printer_channel, (:nonpass_face, :ptr_warn))
         for i in 1:length(phases)
             phase_tests, sem, shared_worker = phases[i]
             isempty(phase_tests) && continue
@@ -1536,11 +1544,12 @@ function _runtests(mod::Module, args::ParsedArgs;
                 # got here normally; there is no point retrying a run being torn down.
                 done[] && break
 
-                retries == i && (io_ctx.nonpass_face[] = :ptr_error)
                 retry_tests = [r.test for r in results.value
                                     if r.result isa Exception || anynonpass(r.result[])]
                 isempty(retry_tests) && break
 
+                # the last attempt of a test is the one that gets reported, so print it red
+                retries == i && put!(printer_channel, (:nonpass_face, :ptr_error))
                 put!(printer_channel, (:retry, length(retry_tests), i))
                 sem = Base.Semaphore(1)
                 shared_worker = serial_worker

From ab7efc9dc94833d5c0bdacdea56cb33e5ba20842 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sat, 15 Aug 2026 07:14:16 -0300
Subject: [PATCH 21/21] Update src/ParallelTestRunner.jl
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com>
---
 src/ParallelTestRunner.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 37a7791..d337f08 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1264,7 +1264,7 @@ function _runtests(mod::Module, args::ParsedArgs;
     # (:finished, test_name, worker_id, record)
     # (:crashed, test_name, worker_id, test_time)
     # (:retry, tests_n, retry_n)
-    # (:nonpass_color, color)
+    # (:nonpass_face, face)
     printer_channel = Channel{Tuple}(100)
 
     printer_task = @async begin