From 68693d130ac2d7767386b7c6888ea5a0e6b4aed4 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 23 Jun 2026 14:35:55 +0000 Subject: [PATCH] Revert "CI A1: infra-vs-structural classifier + green-on-main merge-blocking verdict (single authority, fail-closed) (#5615)" This reverts commit 4fceebfe269dfcd726cb1d71679acaa55bc53ac2. --- .github/workflows/ci.yml | 6 +- docs/plans/ci-merge-freshness.md | 30 ----- dsl/extdeps/cache/sccache.dag | 2 - dsl/extdeps/github/ci_runner.dag | 22 ---- dsl/extdeps/runtime/errno.dag | 29 ---- dsl/extdeps/rust/std_process.dag | 22 ---- dsl/gunbc/ci_failure_class.dag | 111 ---------------- dsl/gunbc/ci_spec.dag | 5 +- .../claim/ci_failure_class_witness_test.dag | 124 ------------------ 9 files changed, 5 insertions(+), 346 deletions(-) delete mode 100644 dsl/extdeps/github/ci_runner.dag delete mode 100644 dsl/extdeps/runtime/errno.dag delete mode 100644 dsl/extdeps/rust/std_process.dag delete mode 100644 dsl/gunbc/ci_failure_class.dag delete mode 100644 dsl/test/claim/ci_failure_class_witness_test.dag diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7915b2ae80b..07d4c8ba495 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: echo "::warning::cargo/sccache EAGAIN under fleet pressure; cold retry CARGO_BUILD_JOBS=1 (keep sccache)" RETRY_LOG=$(mktemp) if ! ( CARGO_BUILD_JOBS=1 cargo build -p v1-compiler --release --features text_lookup_work_counter --bins ) 2>&1 | tee "$RETRY_LOG"; then - if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache: encountered fatal error' "$RETRY_LOG"; then + if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache:' "$RETRY_LOG"; then echo "::warning::cargo/sccache still failing; retry without RUSTC_WRAPPER" env -u RUSTC_WRAPPER CARGO_BUILD_JOBS=1 cargo build -p v1-compiler --release --features text_lookup_work_counter --bins || exit 1 else @@ -144,7 +144,7 @@ jobs: echo "::warning::cargo/sccache EAGAIN under fleet pressure; cold retry CARGO_BUILD_JOBS=1 (keep sccache)" RETRY_LOG=$(mktemp) if ! ( CARGO_BUILD_JOBS=1 cargo build -p v1-compiler --release --features text_lookup_work_counter --bins ) 2>&1 | tee "$RETRY_LOG"; then - if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache: encountered fatal error' "$RETRY_LOG"; then + if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache:' "$RETRY_LOG"; then echo "::warning::cargo/sccache still failing; retry without RUSTC_WRAPPER" env -u RUSTC_WRAPPER CARGO_BUILD_JOBS=1 cargo build -p v1-compiler --release --features text_lookup_work_counter --bins || exit 1 else @@ -170,7 +170,7 @@ jobs: echo "::warning::cargo/sccache EAGAIN under fleet pressure; cold retry CARGO_BUILD_JOBS=1 (keep sccache)" RETRY_LOG=$(mktemp) if ! ( CARGO_BUILD_JOBS=1 "$ROOT/target/release/gunbc" run --source-root dsl --source-root src/v2 --entry dsl/tools/rust_gates_ci.dag --function main ) 2>&1 | tee "$RETRY_LOG"; then - if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache: encountered fatal error' "$RETRY_LOG"; then + if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache:' "$RETRY_LOG"; then echo "::warning::cargo/sccache still failing; retry without RUSTC_WRAPPER" env -u RUSTC_WRAPPER CARGO_BUILD_JOBS=1 "$ROOT/target/release/gunbc" run --source-root dsl --source-root src/v2 --entry dsl/tools/rust_gates_ci.dag --function main || exit 1 else diff --git a/docs/plans/ci-merge-freshness.md b/docs/plans/ci-merge-freshness.md index 62daa5bd29c..ee516690fb2 100644 --- a/docs/plans/ci-merge-freshness.md +++ b/docs/plans/ci-merge-freshness.md @@ -61,33 +61,3 @@ re-leak silently excused, a §5 fail-open). It is **complementary** and explicit 3×-red fix — claiming so would be a false-green about a false-green. The forward completeness checks already exist; the reverse-staleness gate is the genuine residual lens; the freshness requirement in §3 is the actual 3×-red killer. - -## 6. The infra-vs-structural verdict ships DORMANT (scaffold + written enable-gate) - -#5615 (`gunbc.ci_failure_class`) builds the construction-first half that makes the §3 freshness -requirement *viable*: a classifier that partitions a floor red into `Infra | Structural` and a -fail-closed verdict surface `floor_exit_blocks_merge(ProcessExit) -> Bool` (success / unambiguous-infra ⇒ -non-blocking; everything else, including a bare exit-137, ⇒ Structural ⇒ blocks). Without it, requiring -green-on-main would block merges on the environmental reds (sccache/EAGAIN/runner contention) that are -*not* the defect the freshness rule targets. - -**#5615 ships this verdict surface DORMANT.** It is built and witnessed but **nothing consumes -`floor_exit_blocks_merge` to block any merge** — it is dormant by construction, since the freshness -*mechanism* (§3, the operator merge-queue that would actually re-run vs current `main` and consume the -verdict) is explicitly out of #5615's scope. This is a §6 **scaffold-with-dissolution-trigger**: the -verdict exists; flipping it to *enforcing* is the dissolution event, and it is gated — **do not wire the -merge-queue to block on this verdict until BOTH hold:** - -- **(a) the cross-run clustering-reclassification consumer has landed.** A *lone* exit-137 fails closed - to `Structural` (a single `ProcessExit` cannot tell a co-residence OOM from a structural - memory-runaway). If the green-on-main mechanism went live first, a genuine co-residence OOM on a - re-run would read `Structural ⇒ block ⇒ re-run ⇒ OOM ⇒ …` — the livelock deep-otter-528 warned of. - The clustering consumer (simultaneous multi-job death + host-uptime reset in a tight window) is what - *reclassifies* such a clustered OOM back to `Infra`; it is a **hard prerequisite** for enforcement. -- **(b) cap-recovery has reclaimed session slots.** Green-on-main re-runs *increase* floor memory load - during exactly the contention window this lockdown is containing; enforcing before the host has slack - worsens the OOM pressure the verdict is meant to triage. - -Until both clear, the verdict is a loaded mechanism left unwired on purpose: a future worker must **not** -flip it to enforcing early. The trigger is decidable (both consumers either exist on `main` or do not), -so this is a wall-after-grounding, not a forever-ratchet. diff --git a/dsl/extdeps/cache/sccache.dag b/dsl/extdeps/cache/sccache.dag index 5efab079449..ab95ada5b8a 100644 --- a/dsl/extdeps/cache/sccache.dag +++ b/dsl/extdeps/cache/sccache.dag @@ -36,8 +36,6 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { data sccache_local_id: CacheInterfaceId = "sccache_local" -data sccache_fatal_error_log_fragment: String = "sccache: encountered fatal error" - data sccache_local_facts: CacheInterfaceCatalogFacts = { identity: sccache_local_id backing_surface: "sccache-local-disk" diff --git a/dsl/extdeps/github/ci_runner.dag b/dsl/extdeps/github/ci_runner.dag deleted file mode 100644 index 8b61c330e15..00000000000 --- a/dsl/extdeps/github/ci_runner.dag +++ /dev/null @@ -1,22 +0,0 @@ -module extdeps.github.ci_runner - -import extdeps.external_authority { ExternalAuthority } -import extdeps.uri { Uri, Https } - -data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { - uri: Uri { - scheme: Https - locator: "docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners" - } -} - -type RunnerLifecycleSignal - = ShutdownReceived - | OperationCanceled - -fn runner_lifecycle_log_fragment(signal: RunnerLifecycleSignal) -> String { - match signal { - ShutdownReceived => "The runner has received a shutdown signal" - OperationCanceled => "The operation was canceled" - } -} diff --git a/dsl/extdeps/runtime/errno.dag b/dsl/extdeps/runtime/errno.dag deleted file mode 100644 index b55afd84501..00000000000 --- a/dsl/extdeps/runtime/errno.dag +++ /dev/null @@ -1,29 +0,0 @@ -module extdeps.runtime.errno - -import extdeps.external_authority { ExternalAuthority } -import extdeps.uri { Uri, Https } - -data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { - uri: Uri { - scheme: Https - locator: "pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html" - } -} - -type PosixErrno - = EAGAIN - | ENOMEM - -fn errno_number(e: PosixErrno) -> Int { - match e { - EAGAIN => 11 - ENOMEM => 12 - } -} - -fn errno_strerror(e: PosixErrno) -> String { - match e { - EAGAIN => "Resource temporarily unavailable" - ENOMEM => "Cannot allocate memory" - } -} diff --git a/dsl/extdeps/rust/std_process.dag b/dsl/extdeps/rust/std_process.dag deleted file mode 100644 index 2b4064e2f96..00000000000 --- a/dsl/extdeps/rust/std_process.dag +++ /dev/null @@ -1,22 +0,0 @@ -module extdeps.rust.std_process - -import extdeps.external_authority { ExternalAuthority } -import extdeps.uri { Uri, Https } - -data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { - uri: Uri { - scheme: Https - locator: "doc.rust-lang.org/std/process/struct.Command.html" - } -} - -type StdProcessSpawnOutcome - = SpawnSucceeded - | SpawnFailed - -fn spawn_outcome_log_fragment(outcome: StdProcessSpawnOutcome) -> String { - match outcome { - SpawnSucceeded => "" - SpawnFailed => "failed to spawn" - } -} diff --git a/dsl/gunbc/ci_failure_class.dag b/dsl/gunbc/ci_failure_class.dag deleted file mode 100644 index 64052482516..00000000000 --- a/dsl/gunbc/ci_failure_class.dag +++ /dev/null @@ -1,111 +0,0 @@ -module gunbc.ci_failure_class - -import std.process { ProcessExit, ExitSuccess, ExitFailure } -import extdeps.runtime.errno { PosixErrno, EAGAIN, errno_strerror } -import extdeps.rust.std_process { StdProcessSpawnOutcome, SpawnFailed, spawn_outcome_log_fragment } -import extdeps.cache.sccache { sccache_fatal_error_log_fragment } -import extdeps.github.ci_runner { RunnerLifecycleSignal, ShutdownReceived, runner_lifecycle_log_fragment } - -type InfraSignature - = ResourceTemporarilyUnavailable - | ProcessSpawnFailure - | SccacheFatalError - | RunnerLost - -type SignatureSite - = BuildLog - | JobSignal - -type FloorFailureClass - = Infra { signature: InfraSignature } - | Structural { reason: String } - -data verdict_infra_signatures: List = [ - ResourceTemporarilyUnavailable, - ProcessSpawnFailure, - SccacheFatalError, - RunnerLost -] - -fn infra_signature_match_substring(sig: InfraSignature) -> String { - match sig { - ResourceTemporarilyUnavailable => errno_strerror(e: EAGAIN) - ProcessSpawnFailure => spawn_outcome_log_fragment(outcome: SpawnFailed) - SccacheFatalError => sccache_fatal_error_log_fragment - RunnerLost => runner_lifecycle_log_fragment(signal: ShutdownReceived) - } -} - -fn infra_signature_origin(sig: InfraSignature) -> String { - match sig { - ResourceTemporarilyUnavailable => "extdeps.runtime.errno (POSIX EAGAIN, errno 11)" - ProcessSpawnFailure => "extdeps.rust.std_process (Command::spawn failure)" - SccacheFatalError => "extdeps.cache.sccache (server fatal error)" - RunnerLost => "extdeps.github.ci_runner (runner shutdown signal)" - } -} - -fn infra_signature_site(sig: InfraSignature) -> SignatureSite { - match sig { - ResourceTemporarilyUnavailable => BuildLog - ProcessSpawnFailure => BuildLog - SccacheFatalError => BuildLog - RunnerLost => JobSignal - } -} - -fn build_retry_infra_signatures() -> List { - fold(verdict_infra_signatures, init: [], f: fn(acc, sig) { - match infra_signature_site(sig: sig) { - BuildLog => concat(acc, [sig]) - JobSignal => acc - } - }) -} - -fn infra_retry_grep_alternation() -> String { - fold(build_retry_infra_signatures(), init: "", f: fn(acc, sig) { - let sub = infra_signature_match_substring(sig: sig) - if acc == "" { sub } else { concat(concat(acc, "|"), sub) } - }) -} - -fn reason_matches_infra_signature(reason: String, sig: InfraSignature) -> Bool { - string_contains(s: reason, pattern: infra_signature_match_substring(sig: sig)) -} - -fn classify_failure_reason(reason: String) -> FloorFailureClass { - fold(verdict_infra_signatures, init: Structural { reason: reason }, f: fn(acc, sig) { - match acc { - Infra { signature: _ } => acc - Structural { reason: r } => - if reason_matches_infra_signature(reason: r, sig: sig) { - Infra { signature: sig } - } else { - acc - } - } - }) -} - -fn classify_floor_exit(e: ProcessExit) -> FloorFailureClass { - match e { - ExitSuccess => Structural { reason: "" } - ExitFailure { code: _, reason: r } => classify_failure_reason(reason: r) - } -} - -fn floor_failure_blocks_merge(c: FloorFailureClass) -> Bool { - match c { - Infra { signature: _ } => false - Structural { reason: _ } => true - } -} - -fn floor_exit_blocks_merge(e: ProcessExit) -> Bool { - match e { - ExitSuccess => false - ExitFailure { code: _, reason: _ } => - floor_failure_blocks_merge(c: classify_floor_exit(e: e)) - } -} diff --git a/dsl/gunbc/ci_spec.dag b/dsl/gunbc/ci_spec.dag index 3b8578ca7d3..136c0e25ce0 100644 --- a/dsl/gunbc/ci_spec.dag +++ b/dsl/gunbc/ci_spec.dag @@ -6,7 +6,6 @@ import gunbc.ci_layer_roots { } import tools.build_step { BuildArtifact, verifications_script } import extdeps.languages.bash.program { Concat, lit, var_ref } -import gunbc.ci_failure_class { infra_retry_grep_alternation } type Gate = RustMonolithGate @@ -83,7 +82,7 @@ fn ci_cargo_eagain_retry_core(command: String) -> String { concat( concat( concat( - concat(concat(" if grep -qiE '", infra_retry_grep_alternation()), "' \"$BUILD_LOG\"; then\n"), + " if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache: encountered fatal error' \"$BUILD_LOG\"; then\n", " echo \"::warning::cargo/sccache EAGAIN under fleet pressure; cold retry CARGO_BUILD_JOBS=1 (keep sccache)\"\n" ), concat( @@ -94,7 +93,7 @@ fn ci_cargo_eagain_retry_core(command: String) -> String { concat( concat( concat( - concat(concat(" if grep -qiE '", infra_retry_grep_alternation()), "' \"$RETRY_LOG\"; then\n"), + " if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache:' \"$RETRY_LOG\"; then\n", " echo \"::warning::cargo/sccache still failing; retry without RUSTC_WRAPPER\"\n" ), concat(" env -u RUSTC_WRAPPER CARGO_BUILD_JOBS=1 ", concat(command, " || exit 1\n")) diff --git a/dsl/test/claim/ci_failure_class_witness_test.dag b/dsl/test/claim/ci_failure_class_witness_test.dag deleted file mode 100644 index 6512660c1c6..00000000000 --- a/dsl/test/claim/ci_failure_class_witness_test.dag +++ /dev/null @@ -1,124 +0,0 @@ -module test.claim.ci_failure_class_witness - -import std.process { ProcessExit, ExitSuccess, ExitFailure } -import gunbc.ci_failure_class { - InfraSignature, RunnerLost, - FloorFailureClass, Infra, Structural, - classify_failure_reason, classify_floor_exit, - floor_failure_blocks_merge, floor_exit_blocks_merge, - infra_retry_grep_alternation, build_retry_infra_signatures, - verdict_infra_signatures, infra_signature_match_substring, - infra_signature_origin -} - -fn class_is_infra(c: FloorFailureClass) -> Bool { - match c { - Infra { signature: _ } => true - Structural { reason: _ } => false - } -} - -fn class_is_structural(c: FloorFailureClass) -> Bool { - match c { - Infra { signature: _ } => false - Structural { reason: _ } => true - } -} - -fn reason_classifies_infra(reason: String) -> Bool { - class_is_infra(c: classify_failure_reason(reason: reason)) -} - -fn reason_blocks_merge(reason: String) -> Bool { - class_is_structural(c: classify_failure_reason(reason: reason)) - && floor_failure_blocks_merge(c: classify_failure_reason(reason: reason)) - && floor_exit_blocks_merge(e: ExitFailure { code: 1, reason: reason }) -} - -fn list_len(xs: List) -> Int { - fold(xs, init: 0, f: fn(acc, _) { acc + 1 }) -} - -fn list_has_signature(xs: List, target: InfraSignature) -> Bool { - fold(xs, init: false, f: fn(acc, sig) { acc || (sig == target) }) -} - -fn witness_real_infra_reasons_are_non_blocking() -> Bool { - let eagain = "error: failed to run custom build command\nResource temporarily unavailable (os error 11)" - let spawn = "error: could not compile; caused by: failed to spawn process" - let sccache = "sccache: encountered fatal error\nsccache: error: Connection refused (os error 111)" - let runner = "The runner has received a shutdown signal. Stopping current job." - reason_classifies_infra(reason: eagain) - && reason_classifies_infra(reason: spawn) - && reason_classifies_infra(reason: sccache) - && reason_classifies_infra(reason: runner) - && (floor_failure_blocks_merge(c: classify_failure_reason(reason: eagain)) == false) - && (floor_failure_blocks_merge(c: classify_failure_reason(reason: sccache)) == false) -} - -fn witness_structural_not_swallowed_as_infra() -> Bool { - let compile_err = "error[E0599]: no method named `frobnicate` found for type `Node`" - let inference = "type inference failed: cannot unify Int with String at dsl/gunbc/foo.dag:42" - let assertion = "witness layering_imports_clean returned false (1 upward import)" - let layering = "layering_imports gate: upward import std -> extdeps at dsl/std/credentials.dag:7" - let drift = "generated_artifact_drift gate: .github/workflows/ci.yml differs from gunbc ci output" - let oom_structural = "thread 'main' panicked: index out of bounds; aborting (exit code 137)" - reason_blocks_merge(reason: compile_err) - && reason_blocks_merge(reason: inference) - && reason_blocks_merge(reason: assertion) - && reason_blocks_merge(reason: layering) - && reason_blocks_merge(reason: drift) - && reason_blocks_merge(reason: oom_structural) - && class_is_structural(c: classify_floor_exit(e: ExitFailure { code: 137, reason: oom_structural })) - && floor_exit_blocks_merge(e: ExitFailure { code: 137, reason: oom_structural }) -} - -fn witness_bare_exit_137_fails_closed() -> Bool { - let bare = ExitFailure { code: 137, reason: "Process completed with exit code 137." } - class_is_structural(c: classify_floor_exit(e: bare)) - && floor_exit_blocks_merge(e: bare) -} - -fn witness_unknown_reason_fails_closed() -> Bool { - let unknown = "some entirely novel failure mode we have never grounded" - class_is_structural(c: classify_failure_reason(reason: unknown)) - && floor_exit_blocks_merge(e: ExitFailure { code: 1, reason: unknown }) -} - -fn witness_floor_green_does_not_block() -> Bool { - (floor_exit_blocks_merge(e: ExitSuccess) == false) - && class_is_structural(c: classify_floor_exit(e: ExitSuccess)) -} - -fn witness_grep_alternation_is_single_authority() -> Bool { - let expected = "Resource temporarily unavailable|failed to spawn|sccache: encountered fatal error" - (infra_retry_grep_alternation() == expected) - && string_contains(s: infra_retry_grep_alternation(), pattern: "Resource temporarily unavailable|failed to spawn") -} - -fn witness_build_retry_subset_excludes_job_signals() -> Bool { - let build = build_retry_infra_signatures() - (list_len(xs: build) == 3) - && (list_len(xs: verdict_infra_signatures) == 4) - && (list_has_signature(xs: build, target: RunnerLost) == false) - && (string_contains(s: infra_retry_grep_alternation(), pattern: infra_signature_match_substring(sig: RunnerLost)) == false) -} - -fn witness_every_signature_is_grounded() -> Bool { - fold(verdict_infra_signatures, init: true, f: fn(acc, sig) { - acc - && (infra_signature_origin(sig: sig) != "") - && (infra_signature_match_substring(sig: sig) != "") - }) -} - -test fn ci_failure_class_witnesses() -> Bool { - witness_every_signature_is_grounded() - && witness_real_infra_reasons_are_non_blocking() - && witness_structural_not_swallowed_as_infra() - && witness_bare_exit_137_fails_closed() - && witness_unknown_reason_fails_closed() - && witness_floor_green_does_not_block() - && witness_grep_alternation_is_single_authority() - && witness_build_retry_subset_excludes_job_signals() -}