From 0325076b16b4696459e3b5f1847f8e256f0eb39b Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 30 Sep 2026 02:46:21 -0700 Subject: [PATCH] Fix cloud dogfood pause link-down journey Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/cloud-vm-dogfood.yml | 3 +- web/scripts/cloud-vm/dogfood-journey.mjs | 77 ++++++++++++++++++++---- web/scripts/cloud-vm/dogfood-probes.mjs | 5 +- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cloud-vm-dogfood.yml b/.github/workflows/cloud-vm-dogfood.yml index d01267af7fa1..ef87906a63bb 100644 --- a/.github/workflows/cloud-vm-dogfood.yml +++ b/.github/workflows/cloud-vm-dogfood.yml @@ -51,7 +51,7 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ github.sha }} + ref: refs/heads/main persist-credentials: false - name: Setup Bun @@ -113,6 +113,7 @@ jobs: jq -r '.timings | to_entries[] | "| \(.key | sub("Ms$"; "")) | \(.value) |"' "$result" echo jq -r 'if .statusAfterPauseSeries then "Pause status series: " + ([.statusAfterPauseSeries[] | "\(.atMs)ms=\(.status)" + (if .providerStatus then " provider=\(.providerStatus)" else "" end)] | join(", ")) else empty end' "$result" + jq -r 'if .statusAfterPauseLinkDown then "Pause status series (link down): " + ([.statusAfterPauseLinkDown[] | "\(.atMs)ms=\(.status)" + (if .providerStatus then " provider=\(.providerStatus)" else "" end)] | join(", ")) else empty end' "$result" jq -r '.notes[]? | "- \(.)"' "$result" } >> "$GITHUB_STEP_SUMMARY" diff --git a/web/scripts/cloud-vm/dogfood-journey.mjs b/web/scripts/cloud-vm/dogfood-journey.mjs index ad3af32cdecc..4c77cf95fdfa 100644 --- a/web/scripts/cloud-vm/dogfood-journey.mjs +++ b/web/scripts/cloud-vm/dogfood-journey.mjs @@ -110,9 +110,9 @@ function providerStatusFromVm(response) { return undefined; } -async function pauseStatusSeries(vmId, pauseReturnedAt) { +async function pauseStatusSeries(vmId, pauseReturnedAt, targets = [1_000, 5_000, 15_000]) { const series = []; - for (const targetMs of [1_000, 5_000, 15_000]) { + for (const targetMs of targets) { const waitMs = targetMs - (performance.now() - pauseReturnedAt); if (waitMs > 0) await sleep(waitMs); const observedAt = Math.round(performance.now() - pauseReturnedAt); @@ -494,7 +494,7 @@ try { // A freshly enrolled peer can take a moment to reach the provider, and the // hub gives its first handshake about 10 s. - await timed("hub", async () => { + const startHub = async () => { for (let attempt = 1; ; attempt += 1) { hub = startJsonProcess("wg hub", ["wg", "hub", "--config", configPath, "--socket", hubSocket]); try { @@ -507,7 +507,8 @@ try { await sleep(3000); } } - }); + }; + await timed("hub", startHub); // The app asks once so an older daemon is brought to the trusted build; a // fresh image already serves the trusted listener. @@ -531,11 +532,14 @@ try { }); const session = attach.session ?? "cmux"; - const connected = await timed("connect", async () => { - link = startJsonProcess("remote connect", connectArgv(route, session, hubSocket)); + const startLink = async (name = "remote connect") => { + link = startJsonProcess(name, connectArgv(route, session, hubSocket)); return link.waitFor(isConnected, 90_000, "connected snapshot"); + }; + const connected = await timed("connect", async () => { + return startLink(); }); - const localSocket = connected.local_socket; + let localSocket = connected.local_socket; result.transportPath = connected.connection?.transport?.selected_path?.kind ?? null; // The shell computes the suffix, so the echoed command line never matches. @@ -572,6 +576,47 @@ try { })); if (!skipSleep) { + const resumeAnchor = `${marker}-6`; + await typeLine(terminal, `echo ${marker}-$((3+3))`); + await waitForScreen(localSocket, terminal, resumeAnchor, 30_000); + let linkDownOk = true; + + // First remove every source of link traffic, then observe whether the + // provider wakes the machine without a connected client or WireGuard hub. + try { + await link.stop(); + link = undefined; + await hub.stop(); + hub = undefined; + await timed("pauseLinkDown", async () => { + expectStatus(await api("POST", `/api/vm/${encodeURIComponent(vmId)}/pause`, {}, 3 * 60 * 1000), [200, 202], "pause with link down"); + }); + result.statusAfterPauseLinkDown = await pauseStatusSeries(vmId, performance.now(), [1_000, 5_000, 15_000, 30_000]); + } catch (error) { + linkDownOk = false; + note(`pause with link down failed: ${error.message.slice(0, 400)}`); + } finally { + try { + await timed("resumeLinkDown", async () => { + expectStatus(await api("POST", `/api/vm/${encodeURIComponent(vmId)}/resume`, {}, 5 * 60 * 1000), [200], "resume with link down"); + }); + } catch (error) { + linkDownOk = false; + note(`resume with link down failed: ${error.message.slice(0, 400)}`); + } + try { + await timed("relinkAfterLinkDown", async () => { + await startHub(); + const relinked = await startLink("remote connect (after link down)"); + localSocket = relinked.local_socket; + }); + } catch (error) { + linkDownOk = false; + note(`relink after link down failed: ${error.message.slice(0, 400)}`); + } + } + result.linkDownPauseOk = linkDownOk; + const beforePause = link.mark(); await timed("pause", async () => { expectStatus(await api("POST", `/api/vm/${encodeURIComponent(vmId)}/pause`, {}, 3 * 60 * 1000), [200, 202], "pause"); @@ -603,7 +648,7 @@ try { await link.waitFor(isConnected, 120_000, "connected snapshot after resume", beforeResume); }); await timed("terminalAfterResume", async () => { - await waitForScreen(localSocket, terminal, `${marker}-2`, 30_000); + await waitForScreen(localSocket, terminal, resumeAnchor, 30_000); await typeLine(terminal, `echo ${marker}-$((1+2))`); await waitForScreen(localSocket, terminal, `${marker}-3`, 30_000); }); @@ -619,11 +664,11 @@ try { // one dials the same machine with no control-plane call. await link.stop("SIGKILL"); const relaunched = await timed("relaunchReconnect", async () => { - link = startJsonProcess("remote connect (relaunch)", connectArgv(route, session, hubSocket)); - return link.waitFor(isConnected, 90_000, "connected snapshot after relaunch"); + return startLink("remote connect (relaunch)"); }); await timed("restoredTerminal", async () => { - await waitForScreen(relaunched.local_socket, terminal, `${marker}-2`, 30_000); + const anchor = skipSleep ? `${marker}-2` : `${marker}-6`; + await waitForScreen(relaunched.local_socket, terminal, anchor, 30_000); }); await saveScreen(relaunched.local_socket, terminal, "03-after-relaunch"); @@ -642,6 +687,7 @@ try { const failedProbes = ["agentHooks", "agentStatusWorking", "agentStatusIdle", "notifyReachesHost"] .filter((name) => !result[name].ok); + if (result.linkDownPauseOk === false) failedProbes.push("linkDownPause"); const output = summary({ ok: failedProbes.length === 0, ...(failedProbes.length > 0 ? { stage: failedProbes[0], error: `failed probes: ${failedProbes.join(", ")}` } : {}), @@ -652,6 +698,15 @@ try { if (resultFile) writeFileSync(resultFile, `${JSON.stringify(output)}\n`); } catch (error) { const message = error instanceof Error ? error.message : String(error); + if (vmId && authHeaders) { + try { + const deleted = await api("DELETE", `/api/vm/${encodeURIComponent(vmId)}`); + if ([200, 404].includes(deleted.status)) vmId = undefined; + else note(`failure cleanup delete returned ${deleted.status}`); + } catch (cleanupError) { + note(`failure cleanup delete failed: ${cleanupError.message.slice(0, 300)}`); + } + } const output = summary({ ok: false, stage, error: message, ...result }); console.log(JSON.stringify(output, null, 2)); if (resultFile) writeFileSync(resultFile, `${JSON.stringify(output)}\n`); diff --git a/web/scripts/cloud-vm/dogfood-probes.mjs b/web/scripts/cloud-vm/dogfood-probes.mjs index d41dc2b9bf23..1d7de2673976 100644 --- a/web/scripts/cloud-vm/dogfood-probes.mjs +++ b/web/scripts/cloud-vm/dogfood-probes.mjs @@ -65,9 +65,8 @@ async function hookStatus(runtime) { const doneHead = marker; const doneTail = "-done"; const statusCommand = [ - "status=$(cmux --json agent hook status claude codex | tr -d '\\n');", - `printf '%s\\n' \"$status\" | sed -n 's/.*\"provider\":\"claude\"[^}]*\"state\":\"\\([^\"]*\\)\".*/${marker}-claude:\\1/p';`, - `printf '%s\\n' \"$status\" | sed -n 's/.*\"provider\":\"codex\"[^}]*\"state\":\"\\([^\"]*\\)\".*/${marker}-codex:\\1/p'`, + "status=$(cmux --json agent hook status claude codex);", + `printf '%s\\n' \"$status\" | jq -r '.. | objects | select(has(\"provider\") and has(\"state\")) | \"${marker}-\\(.provider):\\(.state)\"'`, `; printf '\\n%s%s\\n' '${doneHead}' '${doneTail}'`, ].join(" "); return probe(runtime, "agentHooks", { providers: { claude: null, codex: null } }, async (deadline, observation) => {