diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index d872aaf92..2df6f5688 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -39,7 +39,7 @@ jobs: # Names this run in the retry ledger written to the job summary (build/RetryLedger.cs). env: CI_JOB_NAME: dotnet-ci - run: ./build.sh ci + run: ./build/run-with-cancellation-relay.sh ci # The CI target now also runs MessageRoutingTests, which boots a real # RabbitMQ broker (started inline by the build target via docker compose). diff --git a/.github/workflows/http.yml b/.github/workflows/http.yml index 206face22..af7e516bc 100644 --- a/.github/workflows/http.yml +++ b/.github/workflows/http.yml @@ -40,13 +40,13 @@ jobs: - name: Run HTTP Tests env: CI_JOB_NAME: CIHttp - run: ./build.sh CIHttp --framework net9.0 + run: ./build/run-with-cancellation-relay.sh CIHttp --framework net9.0 # Asp.Versioning tests are pinned to net10.0 - name: Run HTTP Asp.Versioning Tests env: CI_JOB_NAME: CIHttpAspVersioning - run: ./build.sh CIHttpAspVersioning + run: ./build/run-with-cancellation-relay.sh CIHttpAspVersioning - name: Stop containers if: always() diff --git a/.github/workflows/slow-tests.yml b/.github/workflows/slow-tests.yml index 702fdfbee..57a7ab9fb 100644 --- a/.github/workflows/slow-tests.yml +++ b/.github/workflows/slow-tests.yml @@ -48,7 +48,7 @@ jobs: # Names this run in the retry ledger written to the job summary (build/RetryLedger.cs). env: CI_JOB_NAME: CISlowTests - run: ./build.sh CISlowTests --framework net9.0 + run: ./build/run-with-cancellation-relay.sh CISlowTests --framework net9.0 - name: Stop containers if: always() diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d10215ad..2ecf59685 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -131,7 +131,10 @@ jobs: # Names this job in the retry ledger (build/RetryLedger.cs). GITHUB_JOB is the matrix's # job id -- the same string, "test", for all thirty of these -- so it cannot be used. CI_JOB_NAME: ${{ matrix.target }} - run: ./build.sh ${{ matrix.target }} --framework net9.0 + # Through the relay so a cancellation's SIGTERM reaches the build process's handler + # (build/run-with-cancellation-relay.sh) — the partial ledger and the wedged + # worker's async stacks both depend on it. + run: ./build/run-with-cancellation-relay.sh ${{ matrix.target }} --framework net9.0 # Only reachable when the runner itself survived. If the OOM killer took a *test* process # rather than the runner service, the kill is recorded here and nowhere else. diff --git a/.gitignore b/.gitignore index 6b595263f..24c483f9d 100644 --- a/.gitignore +++ b/.gitignore @@ -206,6 +206,8 @@ artifacts # The previous main run's published test durations, fetched by CI (or by hand) for # Supervisor.KnownTestDurations — see build/TestDurations.cs. previous-durations/ +# Nuke scratch (dotnet installs, the cancellation relay pid handshake) +.nuke/temp/ src/CommonAssemblyInfo.cs paket.lock diff --git a/build/StallCapture.cs b/build/StallCapture.cs index eb74b5bb3..79ff18c25 100644 --- a/build/StallCapture.cs +++ b/build/StallCapture.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Bobcat.Supervisor; +using Nuke.Common.IO; using Serilog; // The successor to build/ci-memory-sampler.sh (GH-4083/GH-4089), retired when Bobcat 0.8.0 moved @@ -54,6 +55,24 @@ IDisposable registerCancellationCapture(Supervisor supervisor, string projectNam // Only under Actions: locally Ctrl-C should stay an ordinary Ctrl-C. if (Environment.GetEnvironmentVariable("GITHUB_ACTIONS") != "true") return null; + // Where the workflow's cancellation relay (build/run-with-cancellation-relay.sh) finds + // us. The runner signals only the step's own shell, and nothing in the bash -> build.sh + // -> `dotnet run` chain forwards SIGTERM to this process — found live on the first + // capped job after the handler shipped: CIKafka wedged, the stall detector named the + // test and its pid, and the partial ledger never happened because the signal never + // arrived here. Publishing the pid lets the relay signal exactly this process. + try + { + var pidFile = RootDirectory / ".nuke" / "temp" / "build.pid"; + pidFile.Parent.CreateDirectory(); + File.WriteAllText(pidFile, Environment.ProcessId.ToString()); + } + catch + { + // Best-effort like everything else here: without the pid the relay just logs that + // it had nothing to signal, and the job degrades to what it did before. + } + var fired = 0; void handle(PosixSignalContext context) diff --git a/build/run-with-cancellation-relay.sh b/build/run-with-cancellation-relay.sh new file mode 100755 index 000000000..1491bc23d --- /dev/null +++ b/build/run-with-cancellation-relay.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Runs ./build.sh while relaying the runner's cancellation signal to the Nuke build process +# itself, so its SIGTERM handler (build/StallCapture.cs) gets to write the partial ledger and +# capture the wedged worker's async stacks before the hard kill. +# +# Why this exists: the runner signals only the step's own shell on cancellation, and nothing in +# the bash -> build.sh -> `dotnet run` chain forwards SIGTERM to the grandchild that registered +# the handler. Found live on the first capped job after the handler shipped: CIKafka wedged, the +# stall detector named the test and pid in the log — and the partial ledger never happened, +# because the signal never arrived. The build process publishes its pid to .nuke/temp/build.pid +# (build/StallCapture.cs), and this wrapper signals exactly that pid — deliberately NOT the +# process group, which would also kill the wedged worker the handler wants to dump. +# +# Usage (in a workflow step): ./build/run-with-cancellation-relay.sh [args...] + +set -uo pipefail + +pid_file=".nuke/temp/build.pid" +rm -f "${pid_file}" + +relay() { + local pid + pid=$(cat "${pid_file}" 2>/dev/null || true) + if [ -n "${pid}" ] && kill -0 "${pid}" 2>/dev/null; then + echo "[stall] relaying cancellation to the build process (pid ${pid})" + kill -TERM "${pid}" 2>/dev/null || true + # Keep the step alive while the handler writes the partial ledger and captures stacks. + # The runner's own hard-kill deadline still bounds everything; this only stops the step + # from exiting underneath the handler. + for _ in $(seq 1 120); do kill -0 "${pid}" 2>/dev/null || break; sleep 1; done + else + echo "[stall] cancellation before the build process published a pid — nothing to relay" + fi +} + +trap relay INT TERM + +./build.sh "$@" & +build_pid=$! +wait "${build_pid}" +status=$? +trap - INT TERM +exit "${status}"