test(e2e): fix EXDEV fixture permissions

src/lib/onboard/sandbox-readiness-tracing.test.ts

@@ -81,7 +81,6 @@ describe("createSandboxReadyWaiter", () => {
       isLinuxDockerDriverGatewayEnabled: () => true,
       now: () => 0,
       sleep,
-      now: () => 0,
     });
 
     await expect(waitForSandboxReady(NAME, 2, 3)).resolves.toEqual({

test/e2e/RETRY_INVENTORY.md

@@ -6,3 +6,4 @@
 | Operation | Owner | Limit | Retry condition | Safety basis | Evidence |
 | --- | --- | --- | --- | --- | --- |
 | `external-gateway-health.tcp-readiness` | `openshell-gateway` | 10 attempts, one second apart | The newly started gateway listener rejects a TCP connection with `ECONNREFUSED` | The probe is read-only. Other errors stop without retry. The Blueprint Runner health operation runs once after the listener opens. | `external-gateway-readiness-retry.json` |
+| `openclaw-plugin-runtime-exdev.onboard-pairing` | `openclaw-plugin-runtime-exdev` | Two attempts | Fresh onboarding reports that canonical CLI device pairing did not appear. | The retry uses `onboard --resume` only when OpenShell reports the sandbox as Ready, the registry resolves its current runtime identity, and the saved in-progress OpenClaw session is paused at `post_verify` for the same sandbox, gateway, and custom Dockerfile. The saved session retains mutation authority. Other errors stop without retry. | `openclaw-plugin-exdev-onboard-retry.json` |

test/e2e/live/openclaw-plugin-runtime-exdev-onboard.ts

@@ -0,0 +1,104 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { type CommandExitResult, outputContainsReadySandbox } from "../fixtures/clients/command.ts";
+import {
+  runBoundedRetry,
+  type BoundedRetryResult,
+  type RetryEvidence,
+} from "../fixtures/retry-policy.ts";
+
+const OPERATION = "openclaw-plugin-runtime-exdev.onboard-pairing";
+const OWNER = "openclaw-plugin-runtime-exdev";
+
+type OnboardPairingRetryOptions<T extends CommandExitResult> = {
+  sandboxName: string;
+  run(attempt: number): Promise<T>;
+  reconcile(value: T | undefined, error: unknown, attempt: number): Promise<boolean>;
+  onEvidence(evidence: RetryEvidence): Promise<void> | void;
+};
+
+type PairingTarget = {
+  gatewayName: string;
+  stateDirectory: string;
+  version: string;
+};
+
+type OnboardPairingSession = {
+  agent: string | null;
+  failure: unknown;
+  machine: { state: string };
+  metadata: { fromDockerfile: string | null; gatewayName: string };
+  resumable: boolean;
+  sandboxName: string | null;
+  status: string;
+};
+
+type OnboardPairingReconciliationOptions<T extends CommandExitResult> = {
+  expectedFromDockerfile: string;
+  sandboxName: string;
+  captureDiagnostics(): Promise<void>;
+  listSandbox(): Promise<T>;
+  loadSession(): OnboardPairingSession | null;
+  resolveTarget(): PairingTarget | null;
+};
+
+export function classifyOpenClawPluginOnboard<T extends CommandExitResult>(
+  value: T | undefined,
+  error: unknown,
+  sandboxName: string,
+):
+  | { outcome: "passed" }
+  | { outcome: "failed"; failureClass: "deterministic" | "transient-external" } {
+  if (error === undefined && value?.exitCode === 0) return { outcome: "passed" };
+  const message = `OpenClaw onboarding for '${sandboxName}' is incomplete because its canonical CLI device pairing did not appear. Resume or rerun onboarding.`;
+  const output = value ? `${value.stdout}\n${value.stderr}` : "";
+  return {
+    outcome: "failed",
+    failureClass:
+      error === undefined && output.includes(message) ? "transient-external" : "deterministic",
+  };
+}
+
+/** Resume once when the startup watcher has not yet published canonical pairing. */
+export function runOpenClawPluginOnboardWithPairingResume<T extends CommandExitResult>(
+  options: OnboardPairingRetryOptions<T>,
+): Promise<BoundedRetryResult<T>> {
+  return runBoundedRetry({
+    operation: OPERATION,
+    owner: OWNER,
+    idempotence: "reconciled-mutation",
+    maxAttempts: 2,
+    run: options.run,
+    classify: (value, error) => classifyOpenClawPluginOnboard(value, error, options.sandboxName),
+    reconcile: options.reconcile,
+    onEvidence: options.onEvidence,
+  });
+}
+
+/** Require the paused finalization session and its live runtime before resume can continue. */
+export async function reconcileOpenClawPluginOnboardPairing<T extends CommandExitResult>(
+  options: OnboardPairingReconciliationOptions<T>,
+): Promise<boolean> {
+  await options.captureDiagnostics();
+  const list = await options.listSandbox();
+  if (list.exitCode !== 0 || !outputContainsReadySandbox(list, options.sandboxName)) return false;
+  try {
+    const target = options.resolveTarget();
+    if (!target) return false;
+    const session = options.loadSession();
+    return (
+      session !== null &&
+      session.status === "in_progress" &&
+      session.resumable === true &&
+      session.failure === null &&
+      session.sandboxName === options.sandboxName &&
+      session.agent === "openclaw" &&
+      session.machine.state === "post_verify" &&
+      session.metadata.gatewayName === target.gatewayName &&
+      session.metadata.fromDockerfile === options.expectedFromDockerfile
+    );
+  } catch {
+    return false;
+  }
+}

test/e2e/live/openclaw-plugin-runtime-exdev-trusted-prebuild.ts

@@ -27,6 +27,64 @@ import {
 
 export const DELEGATED_CAPABILITY_COMMENT_PREFIX =
   "# TEST-ONLY delegated-capability marker from validated canonical OpenShell: ";
+export const TRUSTED_PLUGIN_FIXTURE_IMAGE_DIR =
+  "/usr/local/share/nemoclaw-e2e/weather-plugin";
+
+export function createTrustedPluginFixtureDockerfile(options: {
+  pluginDirName: string;
+  source: string;
+  versionSourceName: string;
+}): string {
+  const runtimeAnchor = "FROM ${BASE_IMAGE}\n";
+  const runtime = options.source.replace(
+    runtimeAnchor,
+    "FROM ${BASE_IMAGE} AS nemoclaw-runtime\n",
+  );
+  const extension = String.raw`
+
+# Build the deterministic custom-plugin fixture used by this live contract.
+FROM builder AS weather-plugin-builder
+WORKDIR /opt/weather
+COPY ${options.pluginDirName}/package.json ${options.pluginDirName}/package-lock.json ${options.pluginDirName}/tsconfig.json ./
+RUN npm ci --ignore-scripts --no-audit --no-fund
+COPY ${options.pluginDirName}/openclaw.plugin.json ./
+COPY ${options.pluginDirName}/src/ ./src/
+COPY ${options.versionSourceName} ./src/version.ts
+RUN npm run build \
+    && npm prune --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund
+
+# Extend the completed managed runtime so its entrypoint, health check, config
+# generation, and permissions remain the source of truth.
+FROM nemoclaw-runtime AS weather-runtime
+ARG NEMOCLAW_TOOL_DISCLOSURE=progressive
+ENV NEMOCLAW_TOOL_DISCLOSURE=${"${NEMOCLAW_TOOL_DISCLOSURE}"}
+COPY --from=weather-plugin-builder --chown=sandbox:sandbox \
+    /opt/weather/package.json \
+    /opt/weather/package-lock.json \
+    /opt/weather/openclaw.plugin.json \
+    ${TRUSTED_PLUGIN_FIXTURE_IMAGE_DIR}/
+COPY --from=weather-plugin-builder --chown=sandbox:sandbox \
+    /opt/weather/dist/ ${TRUSTED_PLUGIN_FIXTURE_IMAGE_DIR}/dist/
+COPY --from=weather-plugin-builder --chown=sandbox:sandbox \
+    /opt/weather/node_modules/ ${TRUSTED_PLUGIN_FIXTURE_IMAGE_DIR}/node_modules/
+
+USER sandbox
+RUN HOME=/sandbox openclaw plugins install ${TRUSTED_PLUGIN_FIXTURE_IMAGE_DIR} \
+    && HOME=/sandbox openclaw plugins enable weather
+
+# Enabling the plugin changes openclaw.json after the managed runtime hashes it.
+# The runtime test copies this fixture from OpenShell's read-only /usr policy tree
+# into tmpfs after the sandbox starts.
+# hadolint ignore=DL3002
+USER root
+RUN chown sandbox:sandbox /sandbox/.openclaw/openclaw.json \
+    && chmod 660 /sandbox/.openclaw/openclaw.json \
+    && sha256sum /sandbox/.openclaw/openclaw.json > /sandbox/.openclaw/.config-hash \
+    && chown sandbox:sandbox /sandbox/.openclaw/.config-hash \
+    && chmod 660 /sandbox/.openclaw/.config-hash
+`;
+  return runtime.trimEnd() + extension;
+}
 
 const TRUSTED_EXDEV_IMAGE_REF_PATTERN = new RegExp(
   `^${LOCAL_SANDBOX_IMAGE_REPO}:[a-z0-9_][a-z0-9_.-]{0,127}$`,

test/e2e/live/openclaw-plugin-runtime-exdev.test.ts

@@ -8,7 +8,9 @@ import os from "node:os";
 import path from "node:path";
 
 import { resolveOpenshell } from "../../../src/lib/adapters/openshell/resolve.ts";
+import { resolveOrdinaryOpenClawPairingTarget } from "../../../src/lib/actions/sandbox/launch-readiness.ts";
 import { pullAndResolveBaseImageDigest } from "../../../src/lib/onboard/base-image.ts";
+import { loadSession } from "../../../src/lib/state/onboard-session.ts";
 import type { ArtifactSink } from "../fixtures/artifacts.ts";
 import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
 import type { CleanupRegistry } from "../fixtures/cleanup.ts";
@@ -21,13 +23,20 @@ import {
 } from "../fixtures/clients/sandbox.ts";
 import { expect, test } from "../fixtures/e2e-test.ts";
 import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts";
+import { captureIssue4462FailureDiagnostics } from "../fixtures/issue-4462-diagnostics.ts";
 import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts";
 import type { TestProgress } from "../fixtures/progress.ts";
 import { parseJsonFromText } from "./json-envelope.ts";
+import {
+  reconcileOpenClawPluginOnboardPairing,
+  runOpenClawPluginOnboardWithPairingResume,
+} from "./openclaw-plugin-runtime-exdev-onboard.ts";
 import {
   buildTrustedPluginFixtureImage,
   createOpenShellTrustedImageWrapper,
+  createTrustedPluginFixtureDockerfile,
   registerTrustedPluginFixtureImageCleanup,
+  TRUSTED_PLUGIN_FIXTURE_IMAGE_DIR,
 } from "./openclaw-plugin-runtime-exdev-trusted-prebuild.ts";
 import {
   type OpenShellComponents,
@@ -40,7 +49,6 @@ import {
 // boundary, and prove it survives restart and recreation.
 
 const WEATHER_FIXTURE_DIR = path.join(REPO_ROOT, "test/e2e/fixtures/plugins/weather");
-const TOOL_DISCLOSURE_ENV_REFERENCE = "${NEMOCLAW_TOOL_DISCLOSURE}";
 const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-oc-exdev";
 const ONBOARD_TIMEOUT_MS = 25 * 60_000;
 const PROBE_TIMEOUT_MS = 60_000;
@@ -185,57 +193,20 @@ function writeCustomPluginVersion(
 function createCustomPluginDockerfile(context: CustomPluginBuildContext): void {
   const sourceDockerfile = path.join(context.sourceRoot, "Dockerfile");
   const source = fs.readFileSync(sourceDockerfile, "utf8");
-  const runtimeAnchor = "FROM ${BASE_IMAGE}\n";
-  const runtime = source.replace(runtimeAnchor, "FROM ${BASE_IMAGE} AS nemoclaw-runtime\n");
-  const pluginDirName = path.basename(context.pluginDirPath);
-  const versionSourceName = path.basename(context.versionSourcePath);
-  const extension = String.raw`
-
-# Build the deterministic custom-plugin fixture used by this live contract.
-FROM builder AS weather-plugin-builder
-WORKDIR /opt/weather
-COPY ${pluginDirName}/package.json ${pluginDirName}/package-lock.json ${pluginDirName}/tsconfig.json ./
-RUN npm ci --ignore-scripts --no-audit --no-fund
-COPY ${pluginDirName}/openclaw.plugin.json ./
-COPY ${pluginDirName}/src/ ./src/
-COPY ${versionSourceName} ./src/version.ts
-RUN npm run build \
-    && npm prune --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund
-
-# Extend the completed managed runtime so its entrypoint, health check, config
-# generation, and permissions remain the source of truth.
-FROM nemoclaw-runtime AS weather-runtime
-ARG NEMOCLAW_TOOL_DISCLOSURE=progressive
-ENV NEMOCLAW_TOOL_DISCLOSURE=${TOOL_DISCLOSURE_ENV_REFERENCE}
-COPY --from=weather-plugin-builder --chown=sandbox:sandbox \
-    /opt/weather/package.json \
-    /opt/weather/package-lock.json \
-    /opt/weather/openclaw.plugin.json \
-    /opt/weather-plugin/
-COPY --from=weather-plugin-builder --chown=sandbox:sandbox \
-    /opt/weather/dist/ /opt/weather-plugin/dist/
-COPY --from=weather-plugin-builder --chown=sandbox:sandbox \
-    /opt/weather/node_modules/ /opt/weather-plugin/node_modules/
-
-USER sandbox
-RUN HOME=/sandbox openclaw plugins install /opt/weather-plugin \
-    && HOME=/sandbox openclaw plugins enable weather
-
-# Enabling the plugin changes openclaw.json after the managed runtime hashes it.
-# hadolint ignore=DL3002
-USER root
-RUN chown sandbox:sandbox /sandbox/.openclaw/openclaw.json \
-    && chmod 660 /sandbox/.openclaw/openclaw.json \
-    && sha256sum /sandbox/.openclaw/openclaw.json > /sandbox/.openclaw/.config-hash \
-    && chown sandbox:sandbox /sandbox/.openclaw/.config-hash \
-    && chmod 660 /sandbox/.openclaw/.config-hash
-`;
   stageWeatherPluginFixture(context);
   writeCustomPluginVersion(context.versionSourcePath, "v1", true);
-  fs.writeFileSync(context.dockerfilePath, runtime.trimEnd() + extension, {
-    encoding: "utf8",
-    flag: "wx",
-  });
+  fs.writeFileSync(
+    context.dockerfilePath,
+    createTrustedPluginFixtureDockerfile({
+      pluginDirName: path.basename(context.pluginDirPath),
+      source,
+      versionSourceName: path.basename(context.versionSourcePath),
+    }),
+    {
+      encoding: "utf8",
+      flag: "wx",
+    },
+  );
 }
 
 type GatewayToolInvocation = {
@@ -288,7 +259,7 @@ async function assertWeatherPluginRuntime(
 const crossDevicePluginInstallSource = `set -eu
 rm -rf ${EXDEV_TMPFS_SOURCE}
 mkdir -p ${EXDEV_TMPFS_SOURCE} /sandbox/.openclaw/extensions
-cp -R /opt/weather-plugin/. ${EXDEV_TMPFS_SOURCE}/
+cp -R ${TRUSTED_PLUGIN_FIXTURE_IMAGE_DIR}/. ${EXDEV_TMPFS_SOURCE}/
 source_device=$(stat -c '%d' ${EXDEV_TMPFS_SOURCE})
 target_device=$(stat -c '%d' /sandbox/.openclaw/extensions)
 printf 'source_device=%s target_device=%s\n' "$source_device" "$target_device"
@@ -515,27 +486,58 @@ test(
       version: "v1",
     });
     openshellWrapper.selectImage(pluginImageV1);
-    const onboard = await host.command(
-      "node",
-      [
-        CLI_ENTRYPOINT,
-        "onboard",
-        "--fresh",
-        "--non-interactive",
-        "--yes-i-accept-third-party-software",
-        "--agent",
-        "openclaw",
-        "--from",
-        customPluginContext.dockerfilePath,
-      ],
-      {
-        artifactName: "openclaw-plugin-exdev-onboard",
-        env: sandboxEnv,
-        timeoutMs: ONBOARD_TIMEOUT_MS,
+    const onboard = await runOpenClawPluginOnboardWithPairingResume({
+      sandboxName: SANDBOX_NAME,
+      run: (attempt) =>
+        host.command(
+          "node",
+          attempt === 1
+            ? [
+                CLI_ENTRYPOINT,
+                "onboard",
+                "--fresh",
+                "--non-interactive",
+                "--yes-i-accept-third-party-software",
+                "--agent",
+                "openclaw",
+                "--from",
+                customPluginContext.dockerfilePath,
+              ]
+            : [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"],
+          {
+            artifactName:
+              attempt === 1
+                ? "openclaw-plugin-exdev-onboard"
+                : "openclaw-plugin-exdev-onboard-pairing-resume",
+            env: sandboxEnv,
+            timeoutMs: ONBOARD_TIMEOUT_MS,
+          },
+        ),
+      reconcile: () =>
+        reconcileOpenClawPluginOnboardPairing({
+          expectedFromDockerfile: customPluginContext.dockerfilePath,
+          sandboxName: SANDBOX_NAME,
+          captureDiagnostics: () =>
+            captureIssue4462FailureDiagnostics(sandbox, {
+              env: sandboxEnv,
+              redactionValues: ["nemoclaw-exdev-dummy-key"],
+              sandboxName: SANDBOX_NAME,
+            }),
+          listSandbox: () =>
+            sandbox.list({
+              artifactName: "openclaw-plugin-exdev-onboard-pairing-reconcile",
+              env: sandboxEnv,
+              timeoutMs: PROBE_TIMEOUT_MS,
+            }),
+          loadSession,
+          resolveTarget: () => resolveOrdinaryOpenClawPairingTarget(SANDBOX_NAME),
+        }),
+      onEvidence: async (evidence) => {
+        await artifacts.writeJson("openclaw-plugin-exdev-onboard-retry.json", evidence);
       },
-    );
-    const onboardText = resultText(onboard);
-    expect(onboard.exitCode, onboardText).toBe(0);
+    });
+    const onboardText = onboard.value ? resultText(onboard.value) : "onboard returned no result";
+    expect(onboard.outcome, onboardText).toBe("passed");
     const weatherAfterOnboard = await assertWeatherPluginRuntime(sandbox, "after-onboard", "v1");
 
     progress.phase("install plugin v1 across filesystems");
@@ -605,7 +607,7 @@ test(
 
     await artifacts.target.complete({
       id: "openclaw-plugin-runtime-exdev",
-      onboardExitCode: onboard.exitCode,
+      onboardExitCode: onboard.value!.exitCode,
       crossDeviceInstallExitCode: crossDeviceInstall.exitCode,
       restartExitCode: restart.exitCode,
       recreateExitCode: recreate.exitCode,