ci(P1): cold-compile — sccache, unified release build, EAGAIN retry

.github/workflows/ci.yml

@@ -24,7 +24,13 @@ jobs:
         run: |
           echo "CARGO_HOME=$RUNNER_TEMP/cargo" >> "$GITHUB_ENV"
           echo "RUSTUP_HOME=$RUNNER_TEMP/rustup" >> "$GITHUB_ENV"
-          echo "CARGO_BUILD_JOBS=2" >> "$GITHUB_ENV"
+          if sccache --show-stats >/dev/null 2>&1; then
+            echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV"
+            echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV"
+            echo "CARGO_BUILD_JOBS=1" >> "$GITHUB_ENV"
+          else
+            echo "CARGO_BUILD_JOBS=2" >> "$GITHUB_ENV"
+          fi
       - name: Checkout
         uses: actions/checkout@v5
         with:
@@ -52,5 +58,27 @@ jobs:
             cargo-ci-${{ runner.os }}-${{ runner.arch }}-
       - name: gunbc ci (.dag witnesses + gates)
         run: |
+          ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
+          set -o pipefail
+          BUILD_LOG=$(mktemp)
+          if ! (cargo build -p v1-compiler --release --features text_lookup_work_counter --bins) 2>&1 | tee "$BUILD_LOG"; then
+            if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache: encountered fatal error' "$BUILD_LOG"; then
+              echo "::warning::cargo/sccache EAGAIN under fleet pressure; cold retry CARGO_BUILD_JOBS=1 (keep sccache)"
+              RETRY_LOG=$(mktemp)
+              if ! ( CARGO_BUILD_JOBS=1 cargo build -p v1-compiler --release --features text_lookup_work_counter --bins ) 2>&1 | tee "$RETRY_LOG"; then
+                if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache:' "$RETRY_LOG"; then
+                  echo "::warning::cargo/sccache still failing; retry without RUSTC_WRAPPER"
+                  env -u RUSTC_WRAPPER CARGO_BUILD_JOBS=1 cargo build -p v1-compiler --release --features text_lookup_work_counter --bins || exit 1
+                else
+                  exit 1
+                fi
+              fi
+              rm -f "$RETRY_LOG"
+            else
+              exit 1
+            fi
+          fi
+          rm -f "$BUILD_LOG"
+          sccache --show-stats 2>/dev/null || true
           git fetch --no-tags origin main 2>/dev/null || true
-          cargo run -p v1-compiler --release --bin gunbc -- ci
+          "$ROOT/target/release/claim_executor" --source-root src/v2 --source-root dsl --plan-entry src/v2/workflow/ci_floor_plan.dag --plan-function gunbc_ci_floor_batches --notice-title "v2 claim corpus"

dsl/gunbc/ci_spec.dag

@@ -73,6 +73,83 @@ data gunbc_ci_spec: CiSpec = {
   notice_title: "v2 claim corpus"
 }
 
+// Unify v1-compiler feature set across CI cargo invocations. v1-compiler-tests enables
+// text_lookup_work_counter; building gunbc/claim_executor without it forces a second full
+// v1-compiler compile when rust gates run clippy --all-targets (~49% CI wall-time prong).
+data ci_release_features: String = "text_lookup_work_counter"
+
+fn ci_repo_root_shell() -> String {
+  "$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
+}
+
+fn ci_release_build_line() -> String {
+  concat(
+    concat(
+      "cargo build -p v1-compiler --release --features ",
+      ci_release_features
+    ),
+    " --bins"
+  )
+}
+
+// Two-tier cold retry on fleet EAGAIN (#4978 / #5138): first CARGO_BUILD_JOBS=1 with
+// sccache (populate daemon when it works); if sccache still cannot spawn rustc, drop
+// RUSTC_WRAPPER and compile direct. Then print stats.
+fn ci_release_build_script() -> String {
+  let build = ci_release_build_line()
+  concat(
+    concat(
+      concat(
+        concat("ROOT=", ci_repo_root_shell(), "\n"),
+        "set -o pipefail\n"
+      ),
+      concat(
+        concat("BUILD_LOG=$(mktemp)\n", "if ! ("),
+        concat(build, ") 2>&1 | tee \"$BUILD_LOG\"; then\n")
+      )
+    ),
+    concat(
+      concat(
+        concat(
+          "  if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache: encountered fatal error' \"$BUILD_LOG\"; then\n",
+          "    echo \"::warning::cargo/sccache EAGAIN under fleet pressure; cold retry CARGO_BUILD_JOBS=1 (keep sccache)\"\n"
+        ),
+        concat(
+          concat("    RETRY_LOG=$(mktemp)\n", "    if ! ( CARGO_BUILD_JOBS=1 "),
+          concat(build, " ) 2>&1 | tee \"$RETRY_LOG\"; then\n")
+        )
+      ),
+      concat(
+        concat(
+          concat(
+            "      if grep -qiE 'Resource temporarily unavailable|failed to spawn|sccache:' \"$RETRY_LOG\"; then\n",
+            "        echo \"::warning::cargo/sccache still failing; retry without RUSTC_WRAPPER\"\n"
+          ),
+          concat("        env -u RUSTC_WRAPPER CARGO_BUILD_JOBS=1 ", concat(build, " || exit 1\n"))
+        ),
+        concat(
+          concat("      else\n", "        exit 1\n"),
+          concat("      fi\n", "    fi\n")
+        )
+      )
+    ),
+    concat(
+      concat(
+        concat("    rm -f \"$RETRY_LOG\"\n", "  else\n"),
+        concat("    exit 1\n", "  fi\n")
+      ),
+      concat(
+        concat("fi\n", "rm -f \"$BUILD_LOG\"\n"),
+        concat("sccache --show-stats 2>/dev/null || true", "")
+      )
+    )
+  )
+}
+
+fn claim_executor_bin_shell() -> String {
+  "\"$ROOT/target/release/claim_executor\""
+}
+
 fn join_slash(parts: List<String>) -> String {
   fold(parts, init: "", f: (acc, p) => if acc == "" { p } else { concat(concat(acc, "/"), p) })
 }
@@ -110,30 +187,27 @@ data floor_plan_function: String = "gunbc_ci_floor_batches"
 
 fn scheduler_invoke(spec: CiSpec) -> String {
   concat(
-    "cargo build -q -p v1-compiler --release --bin claim_executor --bin yaml_check && ",
     concat(
       concat(
-        concat(
-          concat("cargo run -p v1-compiler --release --bin claim_executor --", witness_roots_source_flags(roots: spec.witness_roots)),
-          concat(" --plan-entry ", floor_plan_entry)
-        ),
+        concat(concat(claim_executor_bin_shell(), witness_roots_source_flags(roots: spec.witness_roots)), concat(" --plan-entry ", floor_plan_entry)),
         concat(" --plan-function ", floor_plan_function)
       ),
       concat(concat(" --notice-title \"", spec.notice_title), "\"")
-    )
+    ),
+    ""
   )
 }
 
 fn gunbc_ci_run_script(spec: CiSpec) -> String {
   concat(
-    concat(git_fetch_script(policy: spec.diff_policy), "\n"),
+    concat(
+      concat(ci_release_build_script(), "\n"),
+      concat(git_fetch_script(policy: spec.diff_policy), "\n")
+    ),
     scheduler_invoke(spec: spec)
   )
 }
 
 fn gunbc_ci_workflow_run(spec: CiSpec) -> String {
-  concat(
-    concat(git_fetch_script(policy: spec.diff_policy), "\n"),
-    "cargo run -p v1-compiler --release --bin gunbc -- ci"
-  )
+  gunbc_ci_run_script(spec: spec)
 }

dsl/gunbc/ci_workflow.dag

@@ -57,15 +57,35 @@ data ci_workflow: Workflow = {
       RunStep {
         name: Present { value: "Isolate toolchain dirs" },
         id: none,
+        // CARGO_HOME/RUSTUP_HOME before Setup Rust + Cache Cargo. Fleet sccache is
+        // host-managed (SCCACHE_SERVER_UDS); enable wrapper only when daemon responds.
         run: concat(
           concat(
             concat(
               "echo \"CARGO_HOME=$RUNNER_TEMP/cargo\" >> \"$GITHUB_ENV\"\n",
               "echo \"RUSTUP_HOME=$RUNNER_TEMP/rustup\" >> \"$GITHUB_ENV\"\n"
             ),
-            "echo \"CARGO_BUILD_JOBS=2\" >> \"$GITHUB_ENV\""
+            "if sccache --show-stats >/dev/null 2>&1; then\n"
           ),
-          ""
+          concat(
+            concat(
+              "  echo \"RUSTC_WRAPPER=sccache\" >> \"$GITHUB_ENV\"\n",
+              concat(
+                "  echo \"CARGO_INCREMENTAL=0\" >> \"$GITHUB_ENV\"\n",
+                concat(
+                  "  echo \"CARGO_BUILD_JOBS=1\" >> \"$GITHUB_ENV\"\n",
+                  concat(
+                    "else\n",
+                    concat(
+                      "  echo \"CARGO_BUILD_JOBS=2\" >> \"$GITHUB_ENV\"\n",
+                      "fi"
+                    )
+                  )
+                )
+              )
+            ),
+            ""
+          )
         ),
         shell: none,
         env: none,

dsl/test/claim/ci_spec_witness.dag

@@ -9,8 +9,9 @@ module test.claim.ci_spec_witness
 
 import gunbc.ci_spec {
   CiSpec, WitnessRoot, DiffPolicy, DiffMergeBase,
-  gunbc_ci_spec,
-  scheduler_invoke, gunbc_ci_run_script, git_fetch_script
+  gunbc_ci_spec, ci_release_features,
+  scheduler_invoke, gunbc_ci_run_script, gunbc_ci_workflow_run, git_fetch_script,
+  ci_release_build_script
 }
 
 data spec_alternate_root: CiSpec = {
@@ -82,7 +83,37 @@ fn witness_run_script_invokes_scheduler() -> Bool {
   string_contains(s: script, pattern: git_fetch_script(policy: gunbc_ci_spec.diff_policy)) &&
     string_contains(s: script, pattern: scheduler_invoke(spec: gunbc_ci_spec)) &&
     string_contains(s: script, pattern: "--plan-function gunbc_ci_floor_batches") &&
-    !string_contains(s: script, pattern: "--bin claim_batch")
+    !string_contains(s: script, pattern: "--bin claim_batch") &&
+    !string_contains(s: script, pattern: "cargo run -p v1-compiler --release --bin claim_executor")
+}
+
+fn witness_release_build_unifies_features() -> Bool {
+  string_contains(
+    s: ci_release_build_script(),
+    pattern: concat("--features ", ci_release_features)
+  )
+}
+
+fn witness_release_build_eagain_cold_retry() -> Bool {
+  let script = ci_release_build_script()
+  string_contains(s: script, pattern: "Resource temporarily unavailable|failed to spawn") &&
+    string_contains(s: script, pattern: "CARGO_BUILD_JOBS=1") &&
+    string_contains(s: script, pattern: "keep sccache") &&
+    string_contains(s: script, pattern: "without RUSTC_WRAPPER") &&
+    string_contains(s: script, pattern: "env -u RUSTC_WRAPPER") &&
+    string_contains(s: script, pattern: "sccache --show-stats")
+}
+
+fn witness_scheduler_uses_release_binary() -> Bool {
+  string_contains(
+    s: scheduler_invoke(spec: gunbc_ci_spec),
+    pattern: "$ROOT/target/release/claim_executor"
+  ) &&
+    !string_contains(s: scheduler_invoke(spec: gunbc_ci_spec), pattern: "cargo run")
+}
+
+fn witness_workflow_run_is_collapsed_script() -> Bool {
+  gunbc_ci_workflow_run(spec: gunbc_ci_spec) == gunbc_ci_run_script(spec: gunbc_ci_spec)
 }
 
 fn ci_spec_witnesses() -> Bool {
@@ -91,5 +122,9 @@ fn ci_spec_witnesses() -> Bool {
     witness_scan_subtree_not_in_scheduler_argv() &&
     witness_diff_policy_drives_fetch() &&
     witness_notice_title_from_spec() &&
-    witness_run_script_invokes_scheduler()
+    witness_run_script_invokes_scheduler() &&
+    witness_release_build_unifies_features() &&
+    witness_release_build_eagain_cold_retry() &&
+    witness_scheduler_uses_release_binary() &&
+    witness_workflow_run_is_collapsed_script()
 }

dsl/test/claim/ci_yaml_serializer_witness.dag

@@ -12,6 +12,7 @@ import extdeps.formats.yaml {
 import gunbc.workflow_yaml_project { project_workflow_to_yaml, steps_yaml, jobs_yaml, step_yaml }
 import gunbc.ci_yaml_emit { expected_ci_yml, ci_yml_drifted }
 import gunbc.ci_workflow { ci_workflow }
+import gunbc.ci_spec { gunbc_ci_spec, gunbc_ci_workflow_run }
 
 fn witness_fetch_depth_is_int_not_string() -> Bool {
   let with_block = yaml_mapping(entries: [kv(key: "fetch-depth", value: yaml_int(n: 0))])
@@ -157,6 +158,27 @@ fn witness_restore_keys_content_deeper_than_key() -> Bool {
   )
 }
 
+fn witness_sccache_wired_when_fleet_daemon_up() -> Bool {
+  let yaml = expected_ci_yml()
+  string_contains(s: yaml, pattern: "if sccache --show-stats >/dev/null 2>&1; then") &&
+    string_contains(s: yaml, pattern: "RUSTC_WRAPPER=sccache") &&
+    string_contains(s: yaml, pattern: "CARGO_INCREMENTAL=0") &&
+    string_contains(s: yaml, pattern: "CARGO_BUILD_JOBS=1") &&
+    !string_contains(s: yaml, pattern: "sccache --stop-server") &&
+    !string_contains(s: yaml, pattern: "sccache --start-server") &&
+    !string_contains(s: yaml, pattern: "SCCACHE_DIR=$RUNNER_TEMP/sccache")
+}
+
+// Full-file drift gate (`ci_yml_drifted` == byte-identical to `expected_ci_yml()`); these
+// pin distinctive `gunbc_ci_workflow_run` projections in the committed step body.
+fn witness_gunbc_ci_step_projects_run_script_markers() -> Bool {
+  let yaml = expected_ci_yml()
+  let script = gunbc_ci_workflow_run(spec: gunbc_ci_spec)
+  string_contains(s: yaml, pattern: "env -u RUSTC_WRAPPER") &&
+    string_contains(s: yaml, pattern: "$ROOT/target/release/claim_executor") &&
+    string_contains(s: script, pattern: "env -u RUSTC_WRAPPER")
+}
+
 fn ci_yaml_serializer_keystone_holds() -> Bool {
   witness_fetch_depth_is_int_not_string() &&
     witness_rustflags_empty_string_quoted() &&
@@ -170,5 +192,7 @@ fn ci_yaml_serializer_keystone_holds() -> Bool {
     witness_cache_key_plain_not_quoted() &&
     witness_multiline_without_block_scalar_is_invalid() &&
     witness_cache_path_not_flat_multiline_sibling() &&
-    witness_restore_keys_content_deeper_than_key()
+    witness_restore_keys_content_deeper_than_key() &&
+    witness_sccache_wired_when_fleet_daemon_up() &&
+    witness_gunbc_ci_step_projects_run_script_markers()
 }

dsl/tools/ci_gates.dag

@@ -1,10 +1,16 @@
 // tools/ci_gates.dag — composed effectful CI gates (one resolve, one process).
 //
 // Gate list authority: `gunbc_ci_spec.gates` (not a parallel hardcoded sequence).
+// Production CI runs gates individually via floor_effect_gate_witness adapters;
+// this module remains for direct invocation.
+//
+// dissolve-on: v1 interpreter terminates effectful `fold` over ProcessExit (same
+// class as #5138 floor batch-2 loop). Until then, `main` sequences gates manually
+// (short-circuit on first failure) instead of `fold(spec.gates, …)`.
 
 module tools.ci_gates
 
-import std.process { ProcessExit, ExitSuccess, exit_failure }
+import std.process { ProcessExit, ExitSuccess }
 import gunbc.ci_spec {
   gunbc_ci_spec, CiSpec, Gate,
   RustMonolithGate, EmitHostGate, LayeringImportsGate, DslCompileCleanGate, CiYamlGate,
@@ -32,20 +38,16 @@ fn run_spec_gate(g: Gate) -> ProcessExit {
   }
 }
 
-func run_ci_gates(spec: CiSpec) -> ProcessExit {
-  let all_ok = fold(spec.gates, init: true, f: (acc, g) => {
-    let result = run_spec_gate(g: g)
-    acc && exit_ok(result)
-  })
-  return if all_ok {
-    ExitSuccess
-  } else {
-    exit_failure(
-      reason: "ci gates failed: one or more gates in gunbc_ci_spec.gates"
-    )
-  }
+// Sequential short-circuit — order mirrors `gunbc_ci_spec.gates` until dissolve-on.
+func run_ci_gates_sequential(spec: CiSpec) -> ProcessExit {
+  let g0 = run_spec_gate(g: RustMonolithGate)
+  let g1 = if exit_ok(g0) { run_spec_gate(g: EmitHostGate) } else { g0 }
+  let g2 = if exit_ok(g1) { run_spec_gate(g: LayeringImportsGate) } else { g1 }
+  let g3 = if exit_ok(g2) { run_spec_gate(g: DslCompileCleanGate) } else { g2 }
+  let g4 = if exit_ok(g3) { run_spec_gate(g: CiYamlGate) } else { g3 }
+  if exit_ok(g4) { run_spec_gate(g: SourceRootIngestGate) } else { g4 }
 }
 
 func main() -> ProcessExit {
-  run_ci_gates(spec: gunbc_ci_spec)
+  run_ci_gates_sequential(spec: gunbc_ci_spec)
 }