Relay a cancelled job's SIGTERM to the build process

.github/workflows/dotnet.yml

@@ -39,7 +39,7 @@ jobs:
         # Names this run in the retry ledger written to the job summary (build/RetryLedger.cs).
         env:
           CI_JOB_NAME: dotnet-ci
-        run: ./build.sh ci
+        run: ./build/run-with-cancellation-relay.sh ci
 
       # The CI target now also runs MessageRoutingTests, which boots a real
       # RabbitMQ broker (started inline by the build target via docker compose).

.github/workflows/http.yml

@@ -40,13 +40,13 @@ jobs:
       - name: Run HTTP Tests
         env:
           CI_JOB_NAME: CIHttp
-        run: ./build.sh CIHttp --framework net9.0
+        run: ./build/run-with-cancellation-relay.sh CIHttp --framework net9.0
 
       # Asp.Versioning tests are pinned to net10.0
       - name: Run HTTP Asp.Versioning Tests
         env:
           CI_JOB_NAME: CIHttpAspVersioning
-        run: ./build.sh CIHttpAspVersioning
+        run: ./build/run-with-cancellation-relay.sh CIHttpAspVersioning
 
       - name: Stop containers
         if: always()

.github/workflows/slow-tests.yml

@@ -48,7 +48,7 @@ jobs:
         # Names this run in the retry ledger written to the job summary (build/RetryLedger.cs).
         env:
           CI_JOB_NAME: CISlowTests
-        run: ./build.sh CISlowTests --framework net9.0
+        run: ./build/run-with-cancellation-relay.sh CISlowTests --framework net9.0
 
       - name: Stop containers
         if: always()

.github/workflows/tests.yml

@@ -131,7 +131,10 @@ jobs:
           # Names this job in the retry ledger (build/RetryLedger.cs). GITHUB_JOB is the matrix's
           # job id -- the same string, "test", for all thirty of these -- so it cannot be used.
           CI_JOB_NAME: ${{ matrix.target }}
-        run: ./build.sh ${{ matrix.target }} --framework net9.0
+        # Through the relay so a cancellation's SIGTERM reaches the build process's handler
+        # (build/run-with-cancellation-relay.sh) — the partial ledger and the wedged
+        # worker's async stacks both depend on it.
+        run: ./build/run-with-cancellation-relay.sh ${{ matrix.target }} --framework net9.0
 
       # Only reachable when the runner itself survived. If the OOM killer took a *test* process
       # rather than the runner service, the kill is recorded here and nowhere else.

.gitignore

@@ -206,6 +206,8 @@ artifacts
 # The previous main run's published test durations, fetched by CI (or by hand) for
 # Supervisor.KnownTestDurations — see build/TestDurations.cs.
 previous-durations/
+# Nuke scratch (dotnet installs, the cancellation relay pid handshake)
+.nuke/temp/
 src/CommonAssemblyInfo.cs
 paket.lock
 

build/StallCapture.cs

@@ -5,6 +5,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Bobcat.Supervisor;
+using Nuke.Common.IO;
 using Serilog;
 
 // The successor to build/ci-memory-sampler.sh (GH-4083/GH-4089), retired when Bobcat 0.8.0 moved
@@ -54,6 +55,24 @@ IDisposable registerCancellationCapture(Supervisor supervisor, string projectNam
         // Only under Actions: locally Ctrl-C should stay an ordinary Ctrl-C.
         if (Environment.GetEnvironmentVariable("GITHUB_ACTIONS") != "true") return null;
 
+        // Where the workflow's cancellation relay (build/run-with-cancellation-relay.sh) finds
+        // us. The runner signals only the step's own shell, and nothing in the bash -> build.sh
+        // -> `dotnet run` chain forwards SIGTERM to this process — found live on the first
+        // capped job after the handler shipped: CIKafka wedged, the stall detector named the
+        // test and its pid, and the partial ledger never happened because the signal never
+        // arrived here. Publishing the pid lets the relay signal exactly this process.
+        try
+        {
+            var pidFile = RootDirectory / ".nuke" / "temp" / "build.pid";
+            pidFile.Parent.CreateDirectory();
+            File.WriteAllText(pidFile, Environment.ProcessId.ToString());
+        }
+        catch
+        {
+            // Best-effort like everything else here: without the pid the relay just logs that
+            // it had nothing to signal, and the job degrades to what it did before.
+        }
+
         var fired = 0;
 
         void handle(PosixSignalContext context)

build/run-with-cancellation-relay.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Runs ./build.sh while relaying the runner's cancellation signal to the Nuke build process
+# itself, so its SIGTERM handler (build/StallCapture.cs) gets to write the partial ledger and
+# capture the wedged worker's async stacks before the hard kill.
+#
+# Why this exists: the runner signals only the step's own shell on cancellation, and nothing in
+# the bash -> build.sh -> `dotnet run` chain forwards SIGTERM to the grandchild that registered
+# the handler. Found live on the first capped job after the handler shipped: CIKafka wedged, the
+# stall detector named the test and pid in the log — and the partial ledger never happened,
+# because the signal never arrived. The build process publishes its pid to .nuke/temp/build.pid
+# (build/StallCapture.cs), and this wrapper signals exactly that pid — deliberately NOT the
+# process group, which would also kill the wedged worker the handler wants to dump.
+#
+# Usage (in a workflow step): ./build/run-with-cancellation-relay.sh <target> [args...]
+
+set -uo pipefail
+
+pid_file=".nuke/temp/build.pid"
+rm -f "${pid_file}"
+
+relay() {
+  local pid
+  pid=$(cat "${pid_file}" 2>/dev/null || true)
+  if [ -n "${pid}" ] && kill -0 "${pid}" 2>/dev/null; then
+    echo "[stall] relaying cancellation to the build process (pid ${pid})"
+    kill -TERM "${pid}" 2>/dev/null || true
+    # Keep the step alive while the handler writes the partial ledger and captures stacks.
+    # The runner's own hard-kill deadline still bounds everything; this only stops the step
+    # from exiting underneath the handler.
+    for _ in $(seq 1 120); do kill -0 "${pid}" 2>/dev/null || break; sleep 1; done
+  else
+    echo "[stall] cancellation before the build process published a pid — nothing to relay"
+  fi
+}
+
+trap relay INT TERM
+
+./build.sh "$@" &
+build_pid=$!
+wait "${build_pid}"
+status=$?
+trap - INT TERM
+exit "${status}"