Bobcat 1.0.0-rc.1

.github/workflows/tests.yml

@@ -58,6 +58,11 @@ env:
   # src/CoreTests/AssemblyInfo.cs and src/Marten.Testing/AssemblyInfo.cs. Change it there.
   NUKE_TELEMETRY_OPTOUT: true
 
+# actions: read lets a test job download the previous master run's duration ledger (below).
+permissions:
+  actions: read
+  contents: read
+
 concurrency:
   group: tests-${{ github.workflow }}-${{ github.ref }}
   # Superseded PR pushes are worth cancelling — 30 jobs per push adds up. A master run is not:
@@ -390,6 +395,24 @@ jobs:
         run: ./build.sh compile-project --project ${{ matrix.project }}
         shell: bash
 
+      # The duration ledger travels as an artifact, not a committed file (JasperFx/bobcat#730,
+      # build/TestLedger.cs): the newest successful master run's `folded-test-ledgers` balances this
+      # run's lanes. Best-effort: with no ledger the supervisor balances by test count and says so.
+      - name: Download the previous run's duration ledger
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          run_id=$(gh run list --repo "${{ github.repository }}" --workflow tests.yml --branch master \
+            --status success --limit 1 --json databaseId --jq '.[0].databaseId')
+          if [ -n "$run_id" ]; then
+            gh run download "$run_id" --repo "${{ github.repository }}" --name folded-test-ledgers \
+              --dir artifacts/previous-test-ledger || echo "run $run_id has no folded-test-ledgers artifact"
+          else
+            echo "no successful master run to take a ledger from"
+          fi
+        shell: bash
+
       - name: test
         run: ./build.sh ${{ matrix.target }}
         shell: bash
@@ -477,9 +500,9 @@ jobs:
           path: ledgers
           merge-multiple: true
 
-      # Every job's folded TestLedger (build/TestLedger.cs), gathered into one artifact. Refreshing
-      # the committed durations under build/test-ledger/ is downloading this from a master run and
-      # copying the files over: each one already holds the committed ledger plus this run.
+      # Every job's folded TestLedger (build/TestLedger.cs), gathered into one artifact. The next
+      # run's test jobs download this from the newest successful master run to balance their lanes;
+      # each file already holds the previous ledger plus this run.
       - name: Publish the folded test ledgers
         uses: actions/upload-artifact@v4
         continue-on-error: true

.nuke/build.schema.json

@@ -163,6 +163,9 @@
         "Framework": {
           "type": "string"
         },
+        "PreviousTestLedger": {
+          "type": "string"
+        },
         "Profile": {
           "type": "string"
         },
@@ -179,9 +182,6 @@
             "null"
           ],
           "format": "int32"
-        },
-        "UpdateTestLedger": {
-          "type": "boolean"
         }
       }
     },

CLAUDE.md

@@ -135,6 +135,8 @@ Test targets run through [Bobcat](https://github.com/JasperFx/bobcat)'s supervis
 
 - **Retries are opt-in, and always in a fresh process.** A failing test is retried only when it
   carries `[Trait("Retry", "N")]` (or `[Trait("Isolated", "true")]`); anything else that fails is red.
+  This is Bobcat's own default policy since 1.0 (bobcat#661); never add `[Trait("RetryWarm", "true")]`,
+  which would retry in the same process, over a half-built store.
   A test that only passes on a retry is reported as flaky — in the log, in the GitHub job summary
   and in the JSON ledger under `artifacts/test-ledger/` — never folded into a clean pass. A test
   earns a Retry trait with evidence from the `flakiness` roll-up, and the trait carries a comment
@@ -151,10 +153,13 @@ Test targets run through [Bobcat](https://github.com/JasperFx/bobcat)'s supervis
   a test that hard-codes `marten_testing` or a port ignores its lane.
 - **Stalls and a heartbeat** (report-only): a test in flight for five minutes is named as
   `[STALLED]`, and a progress line every 30s says how far the run got.
-- **Lane balancing from a committed ledger.** `build/test-ledger/{Project}.{framework}.json` is
-  Bobcat's `TestLedger`; its per-test durations balance the lanes. Every run writes the folded
-  ledger to `artifacts/test-ledger/durations/`; refresh the committed copy deliberately with
-  `--update-test-ledger` (or from a CI artifact) — it never rewrites itself.
+- **Lane balancing from the previous master run's ledger** (a CI artifact, nothing committed). Each
+  test job downloads the `folded-test-ledgers` artifact of the newest successful master run into
+  `artifacts/previous-test-ledger/`; its per-test durations (Bobcat's `TestLedger`, keyed by display
+  name since Bobcat 1.0) balance the lanes, and the run writes the folded ledger to
+  `artifacts/test-ledger/durations/`, which the `flakiness` job gathers for the next run. Locally,
+  `--previous-test-ledger <dir>` points at a downloaded copy; with none, lanes balance by test count.
+  The run report's first line says how many durations matched.
 
 This requires each test project to build as a Microsoft.Testing.Platform host, which
 `<UseMicrosoftTestingPlatformRunner>` in `src/Tests.props` turns on for every project importing it.
@@ -173,9 +178,8 @@ share a test project, so DaemonTests and DaemonTests.ManualOnly take 4.x through
 - Run DaemonTests through `./build.sh test-daemon` or its built executable
   (`src/DaemonTests/bin/Release/net10.0/DaemonTests --filter-class …`). `dotnet test` on an MTP 2.x host
   needs `global.json`'s `"test": { "runner": "Microsoft.Testing.Platform" }`, a repo-wide switch not made here.
-- The rendered specifications print to a terminal; under a captured stdout, set
-  `BOBCAT_SPEC_CONSOLE_FILE=<path>` to get them (`BOBCAT_SPEC_CONSOLE=1` prints nothing when stdout is
-  redirected on alpha.1).
+- The rendered specifications print at exit. In CI, set `BOBCAT_SPEC_CONSOLE_FILE=<path>` to get them as a
+  file; since Bobcat 1.0.0-rc.1 (bobcat#717) a redirected stdout gets them too, instead of nothing.
 
 Default test connection: `Host=localhost;Port=5432;Database=marten_testing;Username=postgres;password=postgres` (`build/build.cs:28`). Override with `marten_testing_database` env var.
 

Directory.Packages.props

@@ -3,8 +3,8 @@
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
 
     <!-- One Bobcat line for the specs (DaemonTests) and the build's supervisor (JasperFx/bobcat#562).
-         1.0.0-alpha.1 is ahead of the rc.1 that bobcat#562 originally planned to wait for. -->
-    <BobcatVersion>1.0.0-alpha.1</BobcatVersion>
+         1.0.0-rc.1 is the release bobcat#562 planned to wait for. -->
+    <BobcatVersion>1.0.0-rc.1</BobcatVersion>
 
     <!-- Bobcat.Xunit 1.0 is built on xunit.v3 4.x over Microsoft.Testing.Platform 2.5.1, and the two
          xunit lines cannot share a test project. The rest of the repo stays on the xunit.v3 3.x pinned

build/RetryLedger.cs

@@ -58,6 +58,7 @@ void recordLedger(string projectName, string framework, SupervisorResults result
             Framework = framework,
             Tests = results.Tests.Count,
             CleanPasses = results.CleanPasses.Count,
+            Skipped = results.Skipped.Count,
             PassedOnRetry = results.PassedOnRetry.Count,
             RetriesPerformed = results.RetriesPerformed,
             Failed = results.Failed.Count,
@@ -208,12 +209,12 @@ static void appendStepSummary(LedgerEntry entry)
             {
                 builder.AppendLine($"### Retry ledger — {entry.Job}");
                 builder.AppendLine();
-                builder.AppendLine("| project | tests | clean | passed on retry | retries | failed | indeterminate |");
-                builder.AppendLine("|---|--:|--:|--:|--:|--:|--:|");
+                builder.AppendLine("| project | tests | clean | skipped | passed on retry | retries | failed | indeterminate |");
+                builder.AppendLine("|---|--:|--:|--:|--:|--:|--:|--:|");
             }
 
             builder.AppendLine(
-                $"| {entry.Project} ({entry.Framework}) | {entry.Tests} | {entry.CleanPasses} | " +
+                $"| {entry.Project} ({entry.Framework}) | {entry.Tests} | {entry.CleanPasses} | {entry.Skipped} | " +
                 $"{mark(entry.PassedOnRetry)} | {mark(entry.RetriesPerformed)} | {mark(entry.Failed)} | " +
                 $"{mark(entry.Indeterminate)} |");
 
@@ -297,6 +298,9 @@ class LedgerEntry
         public string Framework { get; init; }
         public int Tests { get; init; }
         public int CleanPasses { get; init; }
+
+        // Bobcat 1.0 counts a skipped test on its own (bobcat#655): neither a clean pass nor a failure.
+        public int Skipped { get; init; }
         public int PassedOnRetry { get; init; }
         public int RetriesPerformed { get; init; }
         public int Failed { get; init; }

build/SupervisedTests.cs

@@ -23,6 +23,9 @@
 //
 //   - Retries are OPT-IN, and always in a FRESH process. A test is retried only when it carries
 //     [Trait("Retry", "N")] (or [Trait("Isolated", "true")]); everything else that fails, fails.
+//     Since Bobcat 1.0 (bobcat#661) that is Bobcat's DefaultFailurePolicy as it ships: a Retry trait
+//     retries in a fresh process unless the test also says [Trait("RetryWarm", "true")], which
+//     nothing in Marten does. The repo's own fresh-process policy from bobcat#565 is gone.
 //     Up to bobcat#565 every failure here was retried, which made "flaky" and "broken" impossible
 //     to tell apart without opening the ledger. A test earns a Retry trait with evidence: the
 //     flakiness roll-up names every test that only passed on a retry. A retry runs in a fresh
@@ -33,8 +36,8 @@
 //     crashed worker is Indeterminate, with its exit code and stderr; a test in flight past five
 //     minutes is named as STALLED (report-only); Bobcat's own RunReport lands in the job summary.
 //
-//   - Lane balancing from measured durations. The committed per-project TestLedger under
-//     build/test-ledger/ feeds Supervisor.KnownTestDurations, so lanes are balanced
+//   - Lane balancing from measured durations. The previous master run's per-project TestLedger,
+//     fetched as a CI artifact, feeds Supervisor.KnownTestDurations, so lanes are balanced
 //     longest-processing-time-first instead of by test count (TestLedger.cs).
 //
 // Parallelism is still mainly at the JOB level: .github/workflows/tests.yml gives every project its
@@ -121,7 +124,6 @@ bool runSupervised(string projectPath, string framework, int workers)
         };
 
         var ledger = loadTestLedger(projectName, framework);
-        var durations = new LaneDurations(ledger);
 
         var supervisor = new Supervisor(factory)
         {
@@ -143,24 +145,20 @@ bool runSupervised(string projectPath, string framework, int workers)
             StallAction = StallAction.Report,
             HeartbeatInterval = TimeSpan.FromSeconds(30),
 
-            // Longest-processing-time-first lane balancing from the committed ledger, matched to
-            // this run's tests by display name as discovery reports them (TestLedger.cs says why
-            // not by uid). No ledger, or no match, degrades to balancing by test count.
-            TestFilter = durations.Observe,
-            KnownTestDurations = durations.KnownTestDurations,
+            // Longest-processing-time-first lane balancing from the previous run's ledger, keyed by
+            // display name (bobcat#730). No ledger, or no match, degrades to balancing by test
+            // count, and the run report's first lines say how many durations matched.
+            KnownTestDurations = ledger.KnownDurations(),
 
             Log = message => Log.Information("  {Message}", message)
         };
 
-        if (!DisableTestRetry) supervisor.AddFailurePolicy(new RetryOptedInTestsInFreshProcess());
-
         var results = supervisor.Run().GetAwaiter().GetResult();
 
         // Said after the fact because the match can only be made once discovery has run. Without
         // this line a ledger that matches nothing is indistinguishable from one that works.
-        if (workers > 1)
-            Log.Information("  lanes were balanced with {Matched} of {Discovered} test duration(s) from the committed ledger",
-                durations.Matched, durations.Discovered);
+        if (workers > 1 && results.DurationCoverage is { } coverage)
+            Log.Information("  lanes were balanced on the previous run's ledger: {Coverage}", coverage);
 
         recordTestLedger(ledger, projectName, framework, results);
         writeRunReport(projectName, framework, results);
@@ -186,32 +184,6 @@ int workerCountFor(int committed)
         return ceiling;
     }
 
-    /// <summary>
-    /// Retries a failure only when the test opted in with a <c>Retry</c> (or <c>Isolated</c>)
-    /// trait, and always in a FRESH process. Abstains otherwise, so Bobcat's
-    /// <c>DefaultFailurePolicy</c> reports the failure as the failure it is.
-    /// </summary>
-    /// <remarks>
-    /// Not the default policy's own handling of the Retry trait, which retries in the SAME process.
-    /// A Marten test that fails partway through has usually left a store, a schema or a running
-    /// daemon behind, and a warm retry then fails for a reason unrelated to the original defect —
-    /// Wolverine measured exactly that (MySqlTests' end_to_end_from_scratch). xUnit workers have no
-    /// reset bracket of their own, so the fresh process is the bracket.
-    /// </remarks>
-    class RetryOptedInTestsInFreshProcess : IFailurePolicy
-    {
-        public Disposition Decide(AttemptContext attempt)
-        {
-            if (attempt.Succeeded || !attempt.RetriesAvailable) return null;
-
-            var optedIn = attempt.HasTrait(ResilienceTags.Retry) || attempt.Trait(ResilienceTags.Isolated) == "true";
-            if (!optedIn) return null;
-
-            return Disposition.RetryInFreshProcess(
-                $"tagged Retry — attempt {attempt.AttemptNumber + 1} runs in a fresh process");
-        }
-    }
-
     bool report(string projectName, string framework, SupervisorResults results)
     {
         recordLedger(projectName, framework, results);