carry of #27531: gate nightly recovery on tree marker

.github/nightly-step-coverage.json

@@ -63,6 +63,7 @@
         "Install compilation dependencies": {"plumbing": true},
         "Check node is on PATH (cmux-next web bundles)": {"pr": "ci-guards.yml:workflow-guard-tests:Validate universal nightly workflow", "note": "tests/test_ensure_setup_node.py runs scripts/ci/ensure-setup-node.sh against a fake tool cache and dist"},
         "Build the web bundles (cmux-next)": {"pr": "cmux-next.yml:web-bundles:Build the web bundles for this source key"},
+        "Stand in for the acpmux companion while the tree publishes": {"plumbing": true},
         "Fetch the resolved cmux-tui tree companions": {"pr": "cmux-next.yml:checks:Script tests (pin-cmux-tui, embed-cef, ensure-cef store)"},
         "Build in-tree acpmux for the compilation cache": {"pr": "cmux-next.yml:cmux-scheme-compile:Build in-tree acpmux for the app bundle"},
         "Build the OptChat Chief (cmux-next)": {"nightly_only": "cargo build of optchat-chief for the macOS arm64 and x86_64 targets needs the macOS SDK; PR builds skip it (bundle-optchat-chief.sh exits 0 without a binary)"},
@@ -115,6 +116,7 @@
         "Wait through GitHub API rate limits in this job's gh calls": {"pr": "ci-guards.yml:workflow-guard-tests:Validate nightly publish-path gh calls wait through a rate limit"},
         "Validate the saved unsigned nightly source": {"pr": "ci-guards.yml:workflow-guard-tests:Validate unsigned nightly resume provenance and routing"},
         "Resolve the original cmux-tui tree": {"plumbing": true},
+        "Build the tip while its tree publishes": {"pr": "cmux-next.yml:checks:Script tests (pin-cmux-tui, embed-cef, ensure-cef store)"},
         "Resolve cmux-tui client commit": {"pr": "cmux-next.yml:checks:Script tests (pin-cmux-tui, embed-cef, ensure-cef store)"},
         "Choose the build commit": {"plumbing": true},
         "Leave the build slot to a newer nightly-next push": {"plumbing": true}},
@@ -230,6 +232,8 @@
         "Verify recovered DMG artifact": {"nightly_only": "statically verifies the notarized stapled recovered DMG and records its exact digest without launching app code"},
         "Launch smoke from a fresh home": {"nightly_only": "launches the recovered notarized DMG from a fresh home in the console session; only a release run has one (gate tested by tests/test_nightly_next_launch_smoke.py)"}
       },
+      "await-nightly-tree-marker": {
+        "Wait for the build commit's tree-commit marker": {"pr": "cmux-next.yml:checks:Script tests (pin-cmux-tui, embed-cef, ensure-cef store)"}},
       "publish-nightly": {
         "Read the live nightly-next feed before the write": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the published feed read-back and the shipped app's channels"},
         "Read the live nightly-next feed back": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the published feed read-back and the shipped app's channels"},

.github/workflows/cmux-next.yml

@@ -1656,6 +1656,7 @@ jobs:
             bash scripts/cmux-next/tests/pin-cmux-tui-run-check.test.sh
             bash scripts/cmux-next/tests/pin-cmux-tui-probe.test.sh
             bash scripts/cmux-next/tests/pin-cmux-tui-run-fetch.test.sh
+            bash scripts/cmux-next/tests/nightly-tree-at-install.test.sh
             bash scripts/cmux-next/tests/fake-clock.test.sh
             bash scripts/cmux-next/tests/preview.test.sh
             python3 tests/test_intent_familiar_warning.py

scripts/ci/nightly-steps/await-tree-marker.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# nightly.yml await-nightly-tree-marker: "Wait for the build commit's tree-commit marker".
+# NIGHTLY_NEXT_TREE_AT_INSTALL: main's promote guard skipped its tree-commit
+# marker check (G5) for a build that started before its tree was published, so
+# it runs here before publish-nightly: the build commit's marker must exist,
+# name the build's tree key, and name the commit whose client build-nightly-app
+# installed as the tree's publisher. Then the published app carries exactly the
+# tree's client. Read-only: files.cmux.com only.
+# Env: BUILD_SHA, KEY, INSTALLED; CMUX_TREE_MARKER_WAIT_SECONDS (default 3000),
+# CMUX_TREE_MARKER_POLL_SECONDS (default 20), CMUX_TUI_PIN_BASE.
+set -euo pipefail
+[[ "$BUILD_SHA" =~ ^[0-9a-f]{40}$ && "$KEY" =~ ^[0-9a-f]{40}$ && "$INSTALLED" =~ ^[0-9a-f]{40}$ ]] \
+  || { echo "error: the build commit, tree key or installed client commit is missing" >&2; exit 1; }
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+base="${CMUX_TUI_PIN_BASE:-https://files.cmux.com/cmux-tui}"
+marker="$base/tree-commits/$BUILD_SHA.json"
+deadline=$(( $(date +%s) + ${CMUX_TREE_MARKER_WAIT_SECONDS:-3000} ))
+while :; do
+  rc=0
+  bash "$here/scripts/cmux-next/tree-commit-marker.sh" check "$BUILD_SHA" >/dev/null || rc=$?
+  if (( rc == 0 )); then
+    body="$(curl -fsS --proto '=https' --connect-timeout 20 --max-time 60 --retry 3 "$marker?t=$(date +%s)")"
+    read -r key published_by < <(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d["key"], d["publishedBy"])' "$body")
+    [[ "$key" == "$KEY" ]] || { echo "::error title=tree marker::$marker names tree $key, not the build's $KEY; nothing is published"; exit 1; }
+    if [[ "$published_by" != "$INSTALLED" ]]; then
+      echo "::error title=tree marker::tree $KEY was published by $published_by, but this build installed the client of $INSTALLED; nothing is published (the next build takes the published tree)"
+      exit 1
+    fi
+    echo "tree $KEY is published by $INSTALLED, the client this build installed; $marker is written"
+    exit 0
+  fi
+  (( rc == 1 || rc == 3 )) || { echo "error: tree-commit-marker.sh check exited $rc" >&2; exit 1; }
+  (( $(date +%s) < deadline )) || { echo "::error title=tree marker::$marker is not written in time; nothing is published"; exit 1; }
+  sleep "${CMUX_TREE_MARKER_POLL_SECONDS:-20}"
+done

scripts/ci/nightly-steps/recover-list-newer-pending.sh

@@ -16,6 +16,12 @@ candidates="$(python3 scripts/ci/find-nightly-next-recovery.py \
 keep=""
 while IFS=$'\t' read -r run_id head; do
   [ -n "$run_id" ] || continue
+  # NIGHTLY_NEXT_TREE_AT_INSTALL: never a build whose tree marker check did not pass.
+  if ! why="$(bash scripts/ci/nightly-steps/recover-tree-marker-ok.sh "$run_id")"; then
+    echo "::notice title=Recovery refused::run $run_id: $why"
+    echo "- refused run $run_id: $why" >> "$GITHUB_STEP_SUMMARY"
+    continue
+  fi
   if verdict="$(bash scripts/ci/nightly-next-recovery-lineage.sh "$GITHUB_REPOSITORY" "$CHANNEL_RELEASE_TAG" "$head" </dev/null)"; then
     keep="$keep $run_id"
   else

scripts/ci/nightly-steps/recover-tree-marker-ok.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# NIGHTLY_NEXT_TREE_AT_INSTALL: whether nightly-next run <run id> may be recovered.
+# A build made before its cmux-tui tree published is publishable only once its run's
+# await-nightly-tree-marker passed (the marker names its client's commit as the tree's
+# publisher). Exit 0 when that job passed, was skipped or never existed; 1 (printing why)
+# when it failed, was cancelled or has not finished, or the run's jobs cannot be read.
+# Env: GITHUB_REPOSITORY, GH_TOKEN (actions: read).
+set -euo pipefail
+run_id="${1:?usage: recover-tree-marker-ok.sh <run id>}"
+[[ "$run_id" =~ ^[0-9]+$ ]] || { echo "not a run id: $run_id"; exit 1; }
+# One bounded read: a nightly run has fewer than 100 jobs, and filter=latest keeps re-runs out.
+if ! states="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?per_page=100&filter=latest" \
+    --jq '.jobs[] | select(.name == "await-nightly-tree-marker") | .conclusion // "pending"' </dev/null 2>/dev/null)"; then
+  echo "its jobs cannot be read"
+  exit 1
+fi
+bad="$(grep -vxE 'success|skipped' <<<"$states" | sort -u | tr '\n' ' ' || true)"
+[[ -z "${bad// /}" ]] || { echo "its cmux-tui tree marker check is ${bad% }"; exit 1; }

scripts/ci/nightly-steps/resolve-tree-pending.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# nightly.yml resolve-nightly-cmux-tui-client: "Build the tip while its tree publishes".
+# NIGHTLY_NEXT_TREE_AT_INSTALL (nightly.yml env; main's promote guard reads it):
+# a nightly-next tip whose own cmux-tui tree is not published yet (verify-key's
+# clean 404) is built as it is. The app compiles now; build-nightly-app installs
+# the tree's attested client once its manifest is uploaded
+# (scripts/cmux-next/await-tree-client.sh), and publish-nightly waits for the
+# commit's tree-commit marker (await-nightly-tree-marker). A published tree or
+# a read failure leaves the decision to the resolver step after this one.
+# Env: TIP_SHA. Writes tree_pending=, and when true key=, source_commit= and
+# candidates= (the tip and its first-parent ancestors with the same key, oldest
+# first, within the last 40: the oldest is the one whose artifacts run builds it).
+set -euo pipefail
+[[ "$(git rev-parse HEAD)" == "$TIP_SHA" ]] || { echo "error: the checkout is not the tip $TIP_SHA" >&2; exit 1; }
+key="$(python3 scripts/ci/cmux_tui_tree_key.py --version v2 HEAD)"
+[[ "$key" =~ ^[0-9a-f]{40}$ ]] || { echo "error: no tree key for $TIP_SHA" >&2; exit 1; }
+rc=0
+bash scripts/cmux-next/pin-cmux-tui.sh verify-key "$key" >/dev/null || rc=$?
+if (( rc != 1 )); then
+  echo "tree_pending=false" >> "$GITHUB_OUTPUT"
+  if (( rc == 0 )); then echo "tree $key of the tip is published; the resolver takes it"
+  else echo "::notice title=cmux-tui tree::could not tell whether tree $key is published (verify-key exit $rc); the resolver decides"; fi
+  exit 0
+fi
+candidates=()
+while read -r commit; do
+  [[ "$(python3 scripts/ci/cmux_tui_tree_key.py --version v2 "$commit" 2>/dev/null)" == "$key" ]] || break
+  candidates=("$commit" ${candidates[@]+"${candidates[@]}"})
+done < <(git rev-list --first-parent -n 40 HEAD)
+{
+  echo "tree_pending=true"
+  echo "key=$key"
+  echo "source_commit=$TIP_SHA"
+  echo "candidates=${candidates[*]}"
+} >> "$GITHUB_OUTPUT"
+echo "::notice title=cmux-tui tree::tree $key of the tip is not published yet; the app builds now and installs the tree's client once its manifest is uploaded (candidates: ${candidates[*]})"
+{
+  echo "### Nightly build commit"
+  echo "- the tip \`$TIP_SHA\`; its cmux-tui tree \`$key\` is still publishing (NIGHTLY_NEXT_TREE_AT_INSTALL)"
+} >> "$GITHUB_STEP_SUMMARY"

scripts/ci/resume-unsigned-nightly.py

@@ -16,6 +16,7 @@
 from pathlib import Path, PurePosixPath
 import re
 import stat
+import tarfile
 import tempfile
 import zipfile
 
@@ -136,6 +137,32 @@ def restore(receipt, root):
     return version
 
 
+def bundled_client_commit(root):
+    # The uploaded app artifact contains the precompressed unsigned tarball.
+    # Open it only after the authenticated ZIP extraction and reject traversal
+    # or links before deriving the client provenance used by the marker gate.
+    app_root = root / "app"
+    archives = list(app_root.glob("*.tar.gz"))
+    require(len(archives) == 1, "resumed unsigned app has no unique app archive")
+    with tarfile.open(archives[0], "r:gz") as archive:
+        for entry in archive.getmembers():
+            path = PurePosixPath(entry.name)
+            require(not path.is_absolute() and ".." not in path.parts and "\\" not in entry.name and
+                    not entry.issym() and not entry.islnk(), "unsafe unsigned app tar member")
+        archive.extractall(app_root)
+    versions = list(app_root.glob("**/Contents/Resources/bin/cmux-tui.version"))
+    require(len(versions) == 1, "resumed unsigned app has no unique cmux-tui.version")
+    fields = {}
+    for line in versions[0].read_text().splitlines():
+        if "=" in line:
+            key, value = line.split("=", 1)
+            fields[key] = value
+    commit = fields.get("commit", "")
+    require(re.fullmatch(r"[0-9a-f]{40}", commit) is not None,
+            "resumed unsigned app has no valid cmux-tui commit")
+    return commit
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("command", choices=("plan", "restore"))
@@ -149,6 +176,7 @@ def main():
     outputs = {key: receipt[key] for key in ("run_id", "attempt", "build_sha", "build")}
     if args.command == "restore":
         outputs["version"] = restore(receipt, args.out)
+        outputs["cmux_tui_commit"] = bundled_client_commit(args.out)
         (args.out / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")
     with open(os.environ["GITHUB_OUTPUT"], "a") as output:
         for key, value in outputs.items():

scripts/cmux-next/await-tree-client.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# A NEXT nightly that starts before its own cmux-tui tree is published
+# (NIGHTLY_NEXT_TREE_AT_INSTALL in nightly.yml) compiles the app first and then
+# waits here, after xcodebuild, for the client it will install. Prints the
+# commit whose commit-addressed manifest (https://files.cmux.com/cmux-tui/<commit>/manifest.json)
+# to install, the same manifest the nightly would have installed had it waited
+# for the tree before starting:
+#   1. once tree <key> is published and verified, the commit its source.json
+#      names (pin-cmux-tui.sh verify-key: the tree's sha256 is the one that
+#      commit's manifest carries). That is exactly today's resolve.
+#   2. before that, the first of <candidate>... (the build commit and its
+#      first-parent ancestors with the same tree key, oldest first: the one
+#      that changed the key is the one whose artifacts run builds it) whose
+#      manifest exists. A manifest is uploaded last, after every binary, so it
+#      exists only for a complete upload. The install that follows
+#      (install-cmux-tui-client.sh --expected-commit) checks its build-provenance
+#      attestation and every sha256; nothing here is trusted beyond "it exists".
+# publish-nightly runs only once the build commit's tree-commit marker names
+# the printed commit as the tree's publisher (await-nightly-tree-marker), so a
+# client from a commit that did not publish the tree is never published.
+#
+# Fails fast when nothing will publish the tree: every CMUX_TUI_TREE_CLIENT_RUN_CHECK_SECONDS
+# (default 300) it reads each candidate's cmux-tui artifacts runs (pin-cmux-tui.sh runs-state)
+# and then pin-cmux-tui.sh active-publisher <key>, and two answers
+# in a row of "no active cmux-tui artifacts run has this tree" end the wait
+# (an unreadable runs list keeps waiting). Otherwise it waits up to
+# CMUX_TUI_TREE_CLIENT_WAIT_SECONDS (default 2700), checking every
+# CMUX_TUI_TREE_CLIENT_POLL_SECONDS (default 20).
+#
+# Usage: await-tree-client.sh <tree key> <candidate commit>...
+# Env: CMUX_TUI_PIN_BASE (default https://files.cmux.com/cmux-tui), GH_TOKEN
+# (actions: read, for the runs list). Exit 0 with the commit on stdout, 1 when
+# nothing is published in time or nothing will publish it, 2 on bad usage.
+set -euo pipefail
+
+key="${1:-}"
+shift || true
+[[ "$key" =~ ^[0-9a-f]{40}$ ]] || { echo "error: the first argument must be a 40-hex tree key" >&2; exit 2; }
+(( $# > 0 )) || { echo "error: name at least one candidate commit" >&2; exit 2; }
+for candidate in "$@"; do
+  [[ "$candidate" =~ ^[0-9a-f]{40}$ ]] || { echo "error: candidate $candidate is not a 40-hex commit" >&2; exit 2; }
+done
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+base="${CMUX_TUI_PIN_BASE:-https://files.cmux.com/cmux-tui}"
+wait_seconds="${CMUX_TUI_TREE_CLIENT_WAIT_SECONDS:-2700}"
+poll_seconds="${CMUX_TUI_TREE_CLIENT_POLL_SECONDS:-20}"
+check_seconds="${CMUX_TUI_TREE_CLIENT_RUN_CHECK_SECONDS:-300}"
+for value in "$wait_seconds" "$poll_seconds" "$check_seconds"; do
+  [[ "$value" =~ ^[1-9][0-9]*$ ]] || { echo "error: the CMUX_TUI_TREE_CLIENT_*_SECONDS values must be positive whole seconds" >&2; exit 2; }
+done
+
+# 0 when <commit>'s manifest answers 200 (a complete transfer), 1 otherwise.
+manifest_exists() {
+  local code
+  code="$(curl -sS --proto '=https' --connect-timeout 20 --max-time 60 -o /dev/null -w '%{http_code}' \
+    "$base/$1/manifest.json?t=$(date +%s)" 2>/dev/null)" || return 1
+  [[ "$code" == 200 ]]
+}
+
+started=$(date +%s)
+next_check=0
+idle_checks=0
+while :; do
+  rc=0
+  out="$(bash "$script_dir/pin-cmux-tui.sh" verify-key "$key" 2>/dev/null)" || rc=$?
+  if (( rc == 0 )); then
+    commit="$(sed -n 's/^commit=//p' <<<"$out" | head -1)"
+    [[ "$commit" =~ ^[0-9a-f]{40}$ ]] || { echo "error: verify-key printed no commit for tree $key" >&2; exit 1; }
+    echo "tree $key is published by ${commit:0:12}; installing its client" >&2
+    echo "$commit"
+    exit 0
+  fi
+  for candidate in "$@"; do
+    if manifest_exists "$candidate"; then
+      echo "the manifest of ${candidate:0:12} (tree $key) is uploaded after $(( $(date +%s) - started )) s; installing its client before the tree publishes" >&2
+      echo "$candidate"
+      exit 0
+    fi
+  done
+  elapsed=$(( $(date +%s) - started ))
+  if (( elapsed >= next_check )); then
+    next_check=$(( elapsed + check_seconds ))
+    # The candidates' own runs first (a filtered read each), then the newest runs of any
+    # commit with this tree: 50 newer runs elsewhere must not hide a candidate's publisher.
+    publisher=""
+    for candidate in "$@"; do
+      case "$(bash "$script_dir/pin-cmux-tui.sh" runs-state "$candidate" 2>/dev/null || echo unknown)" in
+        active) publisher="$candidate"; break ;;
+        unknown) publisher=unknown ;;
+      esac
+    done
+    if [[ -z "$publisher" ]]; then
+      publisher="$(bash "$script_dir/pin-cmux-tui.sh" active-publisher "$key" 2>/dev/null || echo unknown)"
+    fi
+    case "$publisher" in
+      unknown) idle_checks=0; echo "the cmux-tui artifacts runs could not all be read; waiting on" >&2 ;;
+      "") idle_checks=$(( idle_checks + 1 )); echo "no active cmux-tui artifacts run has tree $key (check $idle_checks of 2)" >&2 ;;
+      *) idle_checks=0; echo "the cmux-tui artifacts run of ${publisher:0:12} is building tree $key" >&2 ;;
+    esac
+    if (( idle_checks >= 2 )); then
+      echo "error: tree $key is not published, no candidate manifest is uploaded, and no active cmux-tui artifacts run builds it; nothing will publish it" >&2
+      exit 1
+    fi
+  fi
+  if (( elapsed >= wait_seconds )); then
+    echo "error: tree $key is not published and no candidate manifest is uploaded after $(( elapsed / 60 )) min" >&2
+    exit 1
+  fi
+  sleep "$poll_seconds"
+done

scripts/cmux-next/pin-cmux-tui.sh

@@ -116,6 +116,9 @@
 #          CMUX_TUI_TREE_TIP_WAIT_SECONDS (3000), for the tip's active publisher; CMUX_TUI_TREE_SAME_PATHS, space-separated
 #          paths, skips a commit whose copy of any of them differs from the tip's;
 #          no Mac Release compile is required: the nightly compiles the app itself)
+#        | runs-state <commit> (its cmux-tui artifacts runs: active, none, unknown, or "ended <conclusions>")
+#        | active-publisher <key> (the head commit of an active cmux-tui artifacts run whose
+#          commit has tree <key>; nothing when none; "unknown" when the runs cannot all be read)
 #        | verify-key <key> (commit=<publishing commit> when tree <key> is published and its
 #          manifest sha256 verifies; exit 1 not published (a 404), 3 a read failure or unverifiable)
 #        | wait (wait for the tree; superseded=true output for a superseded commit)
@@ -1795,6 +1798,14 @@ PY
   verify-key)
     verify_key_command "${1:-}"
     ;;
+  runs-state)
+    [[ "${1:-}" =~ ^[0-9a-f]{40}$ ]] || { echo "error: runs-state takes a 40-hex commit" >&2; exit 2; }
+    artifacts_runs_state "$1"
+    ;;
+  active-publisher)
+    [[ "${1:-}" =~ ^[0-9a-f]{40}$ ]] || { echo "error: active-publisher takes a 40-hex tree key" >&2; exit 2; }
+    active_publisher_for_key "$1"
+    ;;
   resolve-exact)
     resolve_exact_tree
     ;;