ci(e2e): wait for Launchable family update

test/brev-launchable-e2e.test.ts

@@ -26,6 +26,7 @@ function fixture(
     deleteFails?: boolean;
     e2eFails?: boolean;
     imageRepositorySha?: string;
+    missingProvisionReceipt?: boolean;
     omitReceiptField?: "imageName" | "imageRepositorySha" | "project";
     provisionImageRepositorySha?: string;
     provisionSha?: string;
@@ -49,6 +50,10 @@ function fixture(
   fs.mkdirSync(workDir);
 
   executable(path.join(bin, "timeout"), '#!/usr/bin/env bash\nshift\nexec "$@"\n');
+  executable(
+    path.join(bin, "sleep"),
+    '#!/usr/bin/env bash\nprintf "sleep %s\\n" "$*" >> "$FAKE_CALLS"\n',
+  );
   executable(
     path.join(bin, "gh"),
     `#!/usr/bin/env bash
@@ -98,16 +103,20 @@ case "$1" in
       '{workspaces:[{id:"ws-1",name:$name,status:"RUNNING",shell_status:$shell,build_status:$build}]}' > "$FAKE_STATE" ;;
   exec)
     case "$3" in
+      *NEMOCLAW_BOOT_IMAGE*)
+        printf 'NEMOCLAW_BOOT_IMAGE=%s\\n' "$FAKE_BOOT_IMAGE"
+        printf '%s\\n' "$INSTANCE_NAME" ;;
       *repo_clean*)
+        [ "$FAKE_MISSING_PROVISION_RECEIPT" != 1 ] || exit 2
         printf 'NEMOCLAW_IDENTITY='
-        jq -cn --arg bootImage "$FAKE_BOOT_IMAGE" --arg sourcePath "$FAKE_SOURCE_PATH" \
-          --arg repo "$FAKE_REPO_SHA" --arg provision "$FAKE_PROVISION_SHA" \
+        jq -cn --arg sourcePath "$FAKE_SOURCE_PATH" --arg repo "$FAKE_REPO_SHA" \
+          --arg provision "$FAKE_PROVISION_SHA" \
           --arg sourceRepository "$FAKE_SOURCE_REPOSITORY" \
           --arg imageRepositorySha "$FAKE_PROVISION_IMAGE_REPOSITORY_SHA" \
           --argjson schemaVersion "$FAKE_SCHEMA_VERSION" --argjson clean "$FAKE_REPO_CLEAN" \
           --argjson overrides "$FAKE_RUNTIME_OVERRIDES" \
-          '{bootImage:$bootImage,schemaVersion:$schemaVersion,sourceRepository:$sourceRepository,
-            sourcePath:$sourcePath,repoSha:$repo,provisionSha:$provision,
+          '{schemaVersion:$schemaVersion,sourceRepository:$sourceRepository,sourcePath:$sourcePath,
+            repoSha:$repo,provisionSha:$provision,
             imageRepositorySha:$imageRepositorySha,repoClean:$clean,runtimeOverrides:$overrides}'
         printf '%s\\n' "$INSTANCE_NAME" ;;
       *) exit 2 ;;
@@ -146,6 +155,7 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n'
     FAKE_DELETE_FAILS: options.deleteFails ? "1" : "0",
     FAKE_E2E_FAILS: options.e2eFails ? "1" : "0",
     FAKE_IMAGE_REPOSITORY_SHA: options.imageRepositorySha ?? "b".repeat(40),
+    FAKE_MISSING_PROVISION_RECEIPT: options.missingProvisionReceipt ? "1" : "0",
     FAKE_OMIT_RECEIPT_FIELD: options.omitReceiptField ?? "",
     FAKE_PROVISION_IMAGE_REPOSITORY_SHA:
       options.provisionImageRepositorySha ?? options.imageRepositorySha ?? "b".repeat(40),
@@ -182,6 +192,10 @@ describe("focused staging Brev Launchable lane", () => {
     expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0);
     const commands = fs.readFileSync(calls, "utf8");
     expect(commands.match(/\/dispatches/gu)).toHaveLength(1);
+    expect(commands).toContain("sleep 300");
+    expect(commands.indexOf("sleep 300")).toBeLessThan(
+      commands.indexOf("create nclaw-e2e-test-1 --launchable env-staging123"),
+    );
     expect(commands).toContain("create nclaw-e2e-test-1 --launchable env-staging123");
     expect(commands).toContain("ssh preinstalled full-e2e.test.ts");
     expect(commands).not.toContain("nvapi-test-value");
@@ -240,8 +254,16 @@ describe("focused staging Brev Launchable lane", () => {
     expect(fs.readFileSync(unready.calls, "utf8")).not.toMatch(/brev exec|full-e2e\.test\.ts/u);
     expect(fs.existsSync(unready.state)).toBe(false);
 
+    const wrongImage = fixture({
+      bootImage: "projects/brevdevprod/global/images/wrong-image",
+    });
+    const wrongImageResult = run(wrongImage.env);
+    expect(wrongImageResult.status).not.toBe(0);
+    expect(wrongImageResult.stderr).toContain("booted image does not match the producer handoff");
+    expect(fs.readFileSync(wrongImage.calls, "utf8")).not.toContain("full-e2e.test.ts");
+    expect(fs.existsSync(wrongImage.state)).toBe(false);
+
     for (const boot of [
-      fixture({ bootImage: "projects/brevdevprod/global/images/wrong-image" }),
       fixture({ repoSha: "b".repeat(40) }),
       fixture({ provisionSha: "b".repeat(40) }),
       fixture({ provisionImageRepositorySha: "c".repeat(40) }),
@@ -272,6 +294,21 @@ describe("focused staging Brev Launchable lane", () => {
     });
   });
 
+  it("preserves the booted image when the provision receipt is missing", () => {
+    const { calls, env, state, workDir } = fixture({ missingProvisionReceipt: true });
+    const result = run(env);
+    expect(result.status).not.toBe(0);
+    expect(fs.existsSync(state)).toBe(false);
+    expect(fs.readFileSync(calls, "utf8")).not.toContain("full-e2e.test.ts");
+    expect(
+      JSON.parse(fs.readFileSync(path.join(workDir, "launchable-e2e.json"), "utf8")),
+    ).toMatchObject({
+      candidateSha,
+      boot: { bootImage: "projects/brevdevprod/global/images/nemoclaw-test-image" },
+      fullE2e: "pending",
+    });
+  });
+
   it("fails the lane when workspace deletion cannot be verified", () => {
     const { env, state } = fixture({ deleteFails: true });
     const result = run({ ...env, BREV_DELETE_TIMEOUT_SECONDS: "1", POLL_SECONDS: "1" });

tools/e2e/brev-launchable-e2e.sh

@@ -151,7 +151,12 @@ expected_boot_image="projects/$(jq -er .project "$manifest")/global/images/$(jq
 image_repository_sha="$(jq -er .imageRepositorySha "$manifest")"
 rm -rf "$WORK_DIR/handoff"
 
-# The standing Launchable resolves the staging family; the guest must contain the exact clean candidate.
+# The standing Launchable resolves the staging family. Give that reference time to
+# observe the family update before deploying it.
+log "Waiting 300s for the Launchable image family to settle"
+sleep 300
+
+# The guest must boot the exact image and contain the exact clean candidate.
 existing="$(workspace)" || die "Brev workspace inventory failed"
 [ -z "$existing" ] || die "workspace name already exists"
 cleanup_required=1
@@ -172,12 +177,28 @@ jq -e '.status == "RUNNING" and (.shell_status // .shellStatus) == "READY" and
 workspace_id="$(jq -r '.id // ""' <<<"$ready")"
 log "Workspace $INSTANCE_NAME ($workspace_id) is ready"
 
-# Return the booted image and the baked runtime receipt.
+# Record the booted image before reading the baked runtime receipt so a stale
+# Launchable image remains visible when the receipt is absent.
 # The remote shell expands the single-quoted command.
 # shellcheck disable=SC2016
-identity="$(timeout 300s brev exec "$INSTANCE_NAME" 'set -euo pipefail
+boot_image="$(timeout 300s brev exec "$INSTANCE_NAME" 'set -euo pipefail
   boot_image=$(curl -fsS --max-time 10 -H "Metadata-Flavor: Google" \
     http://metadata.google.internal/computeMetadata/v1/instance/image)
+  printf "NEMOCLAW_BOOT_IMAGE=%s\n" "$boot_image"' --host \
+  | sed -n 's/^NEMOCLAW_BOOT_IMAGE=//p' | tail -n 1)"
+[ -n "$boot_image" ] || die "booted image identity is missing"
+
+jq -n --arg candidateSha "$CANDIDATE_SHA" --arg producerRun "$producer_run" \
+  --arg bootImage "$boot_image" --arg workspaceName "$INSTANCE_NAME" --arg workspaceId "$workspace_id" \
+  '{candidateSha:$candidateSha,producer:{runId:$producerRun,status:"success"},boot:{bootImage:$bootImage},workspace:{name:$workspaceName,id:$workspaceId},fullE2e:"pending"}' \
+  >"$WORK_DIR/launchable-e2e.json"
+
+[ "$boot_image" = "$expected_boot_image" ] \
+  || die "booted image does not match the producer handoff"
+
+# Return the baked runtime receipt.
+# shellcheck disable=SC2016
+identity="$(timeout 300s brev exec "$INSTANCE_NAME" 'set -euo pipefail
   schema_version=$(sudo -n jq -er .schemaVersion /etc/nemoclaw/provision.json)
   source_repository=$(sudo -n jq -er .sourceRepository /etc/nemoclaw/provision.json)
   source_path=$(sudo -n jq -er .sourcePath /etc/nemoclaw/provision.json)
@@ -195,28 +216,25 @@ identity="$(timeout 300s brev exec "$INSTANCE_NAME" 'set -euo pipefail
     runtime_overrides=false
   fi
   printf "NEMOCLAW_IDENTITY="
-  jq -cn --arg bootImage "$boot_image" --argjson schemaVersion "$schema_version" \
-    --arg sourceRepository "$source_repository" --arg sourcePath "$source_path" \
+  jq -cn --argjson schemaVersion "$schema_version" --arg sourceRepository "$source_repository" \
+    --arg sourcePath "$source_path" \
     --arg repoSha "$repo_sha" --arg provisionSha "$provision_sha" \
     --arg imageRepositorySha "$image_repository_sha" --argjson repoClean "$repo_clean" \
     --argjson runtimeOverrides "$runtime_overrides" \
-    "{bootImage:\$bootImage,schemaVersion:\$schemaVersion,sourceRepository:\$sourceRepository,sourcePath:\$sourcePath,repoSha:\$repoSha,provisionSha:\$provisionSha,imageRepositorySha:\$imageRepositorySha,repoClean:\$repoClean,runtimeOverrides:\$runtimeOverrides}"' --host \
+    "{schemaVersion:\$schemaVersion,sourceRepository:\$sourceRepository,sourcePath:\$sourcePath,repoSha:\$repoSha,provisionSha:\$provisionSha,imageRepositorySha:\$imageRepositorySha,repoClean:\$repoClean,runtimeOverrides:\$runtimeOverrides}"' --host \
   | sed -n 's/^NEMOCLAW_IDENTITY=//p' | tail -n 1)"
-jq -e --arg sha "$CANDIDATE_SHA" --arg bootImage "$expected_boot_image" \
-  --arg imageRepositorySha "$image_repository_sha" '
-  .bootImage == $bootImage and .schemaVersion == 1 and
-  .sourceRepository == "NVIDIA/NemoClaw" and
+jq -e --arg sha "$CANDIDATE_SHA" --arg imageRepositorySha "$image_repository_sha" '
+  .schemaVersion == 1 and .sourceRepository == "NVIDIA/NemoClaw" and
   .sourcePath == "/opt/nemoclaw-image/NemoClaw" and
   .repoSha == $sha and .provisionSha == $sha and
   .imageRepositorySha == $imageRepositorySha and
   .repoClean == true and .runtimeOverrides == false' \
   <<<"$identity" >/dev/null || die "booted image runtime does not match the producer handoff"
 source_path="$(jq -er .sourcePath <<<"$identity")"
 
-jq -n --arg candidateSha "$CANDIDATE_SHA" --arg producerRun "$producer_run" \
-  --argjson boot "$identity" --arg workspaceName "$INSTANCE_NAME" --arg workspaceId "$workspace_id" \
-  '{candidateSha:$candidateSha,producer:{runId:$producerRun,status:"success"},boot:$boot,workspace:{name:$workspaceName,id:$workspaceId},fullE2e:"pending"}' \
-  >"$WORK_DIR/launchable-e2e.json"
+jq --argjson identity "$identity" '.boot += $identity' \
+  "$WORK_DIR/launchable-e2e.json" >"$WORK_DIR/launchable-e2e.tmp"
+mv "$WORK_DIR/launchable-e2e.tmp" "$WORK_DIR/launchable-e2e.json"
 
 # Run the existing suite from the baked checkout; no source copy, install, or rebuild.
 raw_log="${RUNNER_TEMP:-/tmp}/brev-launchable-e2e-${GITHUB_RUN_ID}.raw"