ci: nextest test pipeline, full-failure signal, PR unthrottle (T2)

.claude/rules/testing.md

@@ -71,6 +71,18 @@ delegation, redaction, retry, or policy behavior survives every wrapper.
 - Extend an existing suite when it owns the same seam.
 - Use `tempfile` for test files and directories. Never hardcode system
   temporary-directory paths.
+- `lock_env()`/`EnvGuard` (`crates/contracts/ironclaw_common/src/env_helpers.rs`,
+  enforced by `scripts/ci/check-hermetic-env.sh`) serializes raw
+  `std::env::set_var`/`remove_var` against sibling tests **that share a
+  process** — the classic `cargo test` thread-per-test model. Under
+  `cargo nextest` (root partitions, crate buckets, uninstrumented
+  integration lanes — see `.config/nextest.toml [profile.ci]`), each test
+  is its own process, so this specific cross-test race cannot happen for
+  nextest-run binaries; the guard is still required for any test that
+  itself spawns multiple threads mutating env within one process. Do not
+  treat the guard's presence as proof of cross-test serialization under
+  nextest, and do not delete it as "now redundant" — confirm which
+  process model actually applies to the binary in question first.
 - Avoid ignored or TODO-pinned tests; landed tests run in CI.
 - Prefer real in-memory implementations, deterministic fakes, or recording
   adapters over mocks that duplicate internal behavior.

.config/nextest.toml

@@ -18,12 +18,64 @@ leak-timeout = "100ms"
 [profile.ci]
 # One flake shouldn't mask other real failures.
 fail-fast = false
+# Determinism over green-washing: no automatic retries in CI.
+retries = 0
 # Inline failures, suppress passing noise, keep Actions logs scannable.
 failure-output = "immediate-final"
 success-output = "never"
-# Stricter CI timeout: a hung test should crash the job fast, not sit
-# against the workflow-level 25-minute cap.
-slow-timeout = { period = "60s", terminate-after = 3 }
+# Per-test hang ceiling: warn every 60s, kill after 10 missed warnings
+# (10m). Replaces the old per-BINARY 28m `timeout` as the hang backstop,
+# catching a stuck test at test granularity instead of job granularity
+# (the 91-122min queue hangs the gate-audit measured were bounded only by
+# a 90m job timeout).
+slow-timeout = { period = "60s", terminate-after = 10 }
+# 2-core/~7GB runners: bound concurrent test PROCESSES to the actual core
+# count. nextest's built-in default already resolves to logical-CPU
+# count, but leaving it implicit makes "RUST_MIN_STACK=64MiB (job env) x
+# concurrent per-test tokio runtimes" an unreviewable number. Pin it.
+test-threads = "num-cpus"
+
+[profile.ci.junit]
+# One JUnit report per nextest invocation. Job steps copy it out before
+# the next invocation in the same step would overwrite it (crate-tests'
+# exact-targets loop runs nextest more than once per step) -- see Tasks
+# 3-5's recipes. Consumed by scripts/ci/junit_summary.py in the required
+# aggregator (Task 1) to report failing TEST names, not just job names.
+path = "junit.xml"
+
+# --- Serialised groups -------------------------------------------------
+#
+# `ironclaw`'s CLI smoke tests boot a real `serve` listener. Their helper
+# picks a port by binding :0, reading the number, then RELEASING it before
+# the child re-binds -- a TOCTOU window the helper's own comment refuses to
+# paper over ("a small serialization fix, not a port-reservation
+# framework -- do not extend it into a pool or retry-with-backoff
+# mechanism"). Under `cargo test` those 20 tests shared one process and the
+# window stayed narrow; nextest runs each test in its own process, widening
+# it until two tests race for the same port and the loser reports
+# "connect to serve listener failed: Connection refused".
+#
+# Observed on this branch: onboard_login_link_then_bearer_authorizes_a_
+# protected_request, 1 of 1417 tests, in the reborn-core bucket.
+#
+# The rule this follows is the plan's own: a nextest-only failure gets
+# serialised here, never "fixed" by editing the test. `max-threads = 1`
+# orders the listener tests relative to each other; `threads-required =
+# "num-test-threads"` also excludes unrelated tests during each listener's
+# bind-close-rebind window. Without that global exclusion, the first fix still
+# failed while ungrouped tests ran concurrently.
+[test-groups]
+cli-serve-listener = { max-threads = 1 }
+
+[[profile.default.overrides]]
+filter = 'package(ironclaw) & binary(smoke) & (test(~serve) | test(~onboard_) | test(~stored_key_reaches) | test(a_real_env_var_beats_the_config_default_end_to_end))'
+test-group = "cli-serve-listener"
+threads-required = "num-test-threads"
+
+[[profile.ci.overrides]]
+filter = 'package(ironclaw) & binary(smoke) & (test(~serve) | test(~onboard_) | test(~stored_key_reaches) | test(a_real_env_var_beats_the_config_default_end_to_end))'
+test-group = "cli-serve-listener"
+threads-required = "num-test-threads"
 
 # Per-test windows for the handful of scenarios that genuinely need more
 # than 60s (live-recorded fixtures + heavy integration).
@@ -34,3 +86,13 @@ slow-timeout = { period = "300s", terminate-after = 2 }
 [[profile.ci.overrides]]
 filter = "test(~e2e_thread_scheduling) | test(~heavy_integration)"
 slow-timeout = { period = "300s", terminate-after = 2 }
+
+# Root/integration-tier suites drive whole Reborn turns; give them a
+# 20-minute per-test ceiling (still far under the old 28m per-binary cap).
+# Must stay BELOW the test-name override above: nextest takes the first
+# matching [[profile.ci.overrides]] block, and a package-level filter
+# would otherwise shadow the two named-test overrides for suites inside
+# ironclaw_integration_tests.
+[[profile.ci.overrides]]
+filter = "package(ironclaw_integration_tests)"
+slow-timeout = { period = "120s", terminate-after = 10 }

.github/workflows/code_style.yml

@@ -213,6 +213,12 @@ jobs:
           python3 scripts/ci/test_check_h2_advisory_exception.py
           python3 scripts/ci/test_cut_ironclaw_release.py
           scripts/ci/test-hermetic-test-process.sh
+          scripts/ci/test-hermetic-nextest-control-gating.sh
+          scripts/ci/test-hermetic-deterministic-suite-runner.sh
+          scripts/ci/test-run-reborn-group-tests.sh
+          scripts/ci/test-write-local-repro-summary.sh
+          python3 scripts/ci/test_nextest_config.py
+          python3 scripts/ci/test_junit_summary.py
           scripts/ci/test-reborn-docker-entrypoint.sh
           # #7144: this 204-test module had never been run by any lane, so five
           # of its assertions had silently drifted out of sync with the code they