diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7e06993a6ec..e2a8b800acc 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -3620,7 +3620,8 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh openshell-credential-generation-window: - needs: generate-matrix + name: OpenShell / Credential Lifetime + needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-credential-generation-window') }} runs-on: ubuntu-latest permissions: @@ -3636,6 +3637,7 @@ jobs: E2E_OBSERVABLE_OUTCOME: "Credential expiry rotation detach and rebuild preserve the intended access window" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window + E2E_MANAGED_IMAGE_REVISION: ${{ needs.generate-matrix.outputs.managed_image_catalog == '' && needs.base-image-publication.outputs.managed_image_revision || '' }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_OPENSHELL_CHANNEL: stable NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1" diff --git a/src/lib/onboard/sandbox-create-plan-materialization.ts b/src/lib/onboard/sandbox-create-plan-materialization.ts index 5cfc1fb7432..d7182c17eb8 100644 --- a/src/lib/onboard/sandbox-create-plan-materialization.ts +++ b/src/lib/onboard/sandbox-create-plan-materialization.ts @@ -16,6 +16,8 @@ import { prepareSandboxGpuRoutePolicies } from "./sandbox-gpu-route-policy"; type PrepareInitialSandboxCreatePolicy = typeof import("./initial-policy").prepareInitialSandboxCreatePolicy; +type PlanHermesPortableInitialSandboxPolicy = + typeof import("./initial-policy").planHermesPortableInitialSandboxPolicy; const DCODE_MCP_SNAPSHOT_TMPFS_MOUNT = { type: "tmpfs", @@ -397,11 +399,15 @@ export function materializeSandboxCreatePlan({ } /** Build the schema-5 create plan without provider, filesystem, Docker, or prebuild effects. */ -export function materializeHermesPortableCreatePlan(input: { - readonly intent: SandboxCreateIntent; - readonly fromRef: string; - readonly policyAuthority: SandboxPolicyAuthority; -}): SandboxCreatePlan { +export function materializeHermesPortableCreatePlan( + input: { + readonly intent: SandboxCreateIntent; + readonly fromRef: string; + readonly policyAuthority: SandboxPolicyAuthority; + }, + planInitialSandboxPolicy: PlanHermesPortableInitialSandboxPolicy = + getHermesPortableInitialSandboxPolicy, +): SandboxCreatePlan { const { intent, fromRef, policyAuthority } = input; if (policyAuthority !== "nemoclaw-managed") { throw new Error("Hermes portable sandbox creation requires NemoClaw-managed policy authority."); @@ -421,7 +427,7 @@ export function materializeHermesPortableCreatePlan(input: { "Hermes portable create intent includes an effect that is not owned by its schema-5 receipt.", ); } - const initialSandboxPolicy = getHermesPortableInitialSandboxPolicy( + const initialSandboxPolicy = planInitialSandboxPolicy( intent.policy.basePolicyPath, [...intent.policy.activeMessagingChannels], { diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 49e3e07d72a..5a066502186 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -68,6 +68,18 @@ const channels = [ const discordProviderName = "sandbox-discord-bridge"; +function prepareDiscordInitialSandboxPolicy( + _basePolicyPath: string, + activeMessagingChannels: string[], +) { + const discordEnabled = activeMessagingChannels.includes("discord"); + return { + policyPath: "/tmp/policy.yaml", + appliedPresets: discordEnabled ? ["discord"] : [], + credentialBindingProviders: discordEnabled ? [discordProviderName] : [], + }; +} + function resolveDiscordCreateIntent(input: { selected: boolean; reusable?: boolean }) { const messagingTokenDefs: MessagingTokenDef[] = [ { @@ -122,6 +134,7 @@ function materializeDiscordCreatePlan( runProviderPreDeleteCleanup: vi.fn(), upsertMessagingProviders: vi.fn(() => [discordProviderName]), getHermesToolGatewayProviderName: vi.fn(), + prepareInitialSandboxCreatePolicy: prepareDiscordInitialSandboxPolicy, ...overrides, }); } @@ -640,6 +653,7 @@ describe("resolveSandboxCreateIntent", () => { }, upsertMessagingProviders: vi.fn((_tokenDefs, options) => { expect(options.revalidatePolicyRequirements).toBe(revalidatePolicyRequirements); + options.revalidatePolicyRequirements?.("upserting providers"); events.push("upsert"); return ["sandbox-telegram-bridge"]; }), @@ -666,6 +680,7 @@ describe("resolveSandboxCreateIntent", () => { expect(events).toEqual([ "revalidate:cleaning up providers", "cleanup", + "revalidate:upserting providers", "upsert", "hermes", ]); @@ -775,11 +790,18 @@ describe("resolveSandboxCreateIntent", () => { policyTier: null, }); - const plan = materializeHermesPortableCreatePlan({ - intent, - fromRef: "ghcr.io/nvidia/nemoclaw/hermes:test", - policyAuthority: "nemoclaw-managed", - }); + const plan = materializeHermesPortableCreatePlan( + { + intent, + fromRef: "ghcr.io/nvidia/nemoclaw/hermes:test", + policyAuthority: "nemoclaw-managed", + }, + () => ({ + policyPath: "nemoclaw-blueprint/policies/openclaw-sandbox.yaml", + appliedPresets: [], + sourceBytes: Buffer.from("sandbox: {}"), + }), + ); const configIndex = plan.createArgs.indexOf("--driver-config-json"); expect(JSON.parse(plan.createArgs[configIndex + 1]!)).toEqual({ diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 3da5163dae4..de43867ef13 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -91,6 +91,7 @@ describe("deferred provider effect authority", () => { runVerifiedSandboxCreateEffects: null, activateDeferredProviderEffects: (revalidate) => { revalidate("cleaning up providers for sandbox 'alpha'"); + events.push("activate"); return ["first", "second"]; }, revalidatePolicyAuthorityBeforeCreate: vi.fn(), @@ -128,6 +129,7 @@ describe("deferred provider effect authority", () => { ); expect(events).toContain("policy: attaching provider 'second' to sandbox 'alpha'"); expect(events).toContain("policy: cleaning up providers for sandbox 'alpha'"); + expect(events).toContain("activate"); expect(events).not.toContain("sandbox provider attach -g nemoclaw alpha second"); }); }); diff --git a/test/credentials/openshell-credential-generation-window.test.ts b/test/credentials/openshell-credential-generation-window.test.ts index be9ff51a642..9fdd4920f3c 100644 --- a/test/credentials/openshell-credential-generation-window.test.ts +++ b/test/credentials/openshell-credential-generation-window.test.ts @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { EventEmitter } from "node:events"; import fs from "node:fs"; +import vm from "node:vm"; import { describe, expect, it } from "vitest"; @@ -18,7 +20,8 @@ import { credentialWindowRequestId, credentialWindowSecrets, OPENSHELL_RETAINED_CREDENTIAL_GENERATIONS, -} from "../e2e/live/openshell-credential-generation-window.ts"; + parseCredentialWindowRequestObservation, +} from "../e2e/live/mcp-bridge-credential-generation-window.ts"; describe("OpenShell exact-main credential generation-window proof", () => { it("crosses the complete upstream retention window with unique scannable values", () => { @@ -43,8 +46,6 @@ describe("OpenShell exact-main credential generation-window proof", () => { expect(script.indexOf(snapshot)).toBeLessThan(script.indexOf("while (Date.now() < deadline")); expect(script).toContain('"^openshell:resolve:env:(v[0-9]{1,20})_" + config.envName + "$"'); expect(script).toContain('authorization: "Bearer " + credentialPlaceholder'); - expect(script).toContain('response.statusCode === 200 ? "allowed" : "denied"'); - expect(script).toContain('outbound.on("error", () => resolve("denied"))'); expect(script).toContain("outbound.setTimeout(30_000"); expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_PATHS.acknowledgement)); expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry)); @@ -52,11 +53,169 @@ describe("OpenShell exact-main credential generation-window proof", () => { expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction)); expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval)); expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.deniedAfterDetach)); - expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart)); + expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd)); expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.stop)); expect(script).not.toContain(MCP_BRIDGE_TEST_CREDENTIALS.generationWindow); }); + it("settles a request when the response closes before ending (#10155)", async () => { + const writes = new Map(); + let control: string = CREDENTIAL_WINDOW_STEPS.deniedAfterDetach; + let stdout = ""; + let stderr = ""; + const script = buildCredentialWindowChildScript({ + mcpUrl: "https://credential-window.example.test/mcp", + maxRuntimeMs: 10_000, + }); + const response = Object.assign(new EventEmitter(), { + readableEnded: false, + resume: () => response, + statusCode: 200, + }); + const https = { + request: (_options: unknown, onResponse: (value: typeof response) => void) => { + const outbound = Object.assign(new EventEmitter(), { + destroy: () => outbound, + end: () => { + onResponse(response); + queueMicrotask(() => response.emit("close")); + return outbound; + }, + setTimeout: () => outbound, + }); + return outbound; + }, + }; + const fileSystem = { + existsSync: () => true, + readFileSync: () => control, + writeFileSync: (file: string, value: string) => { + writes.set(file, value); + control = + file === CREDENTIAL_WINDOW_PATHS.acknowledgement ? CREDENTIAL_WINDOW_STEPS.stop : control; + }, + }; + const builtins = new Map([ + ["node:fs", fileSystem], + ["node:https", https], + ]); + const childProcess = { + env: { + [CREDENTIAL_WINDOW_ENV_NAME]: `openshell:resolve:env:v7_${CREDENTIAL_WINDOW_ENV_NAME}`, + }, + exitCode: undefined as number | undefined, + stderr: { + write: (value: string) => { + stderr += value; + return true; + }, + }, + stdout: { + write: (value: string) => { + stdout += value; + return true; + }, + }, + }; + + await (vm.runInNewContext(script, { + Buffer, + URL, + process: childProcess, + queueMicrotask, + require: (id: string) => + builtins.get(id) ?? + (() => { + throw new Error(`unexpected child require: ${id}`); + })(), + setTimeout: (callback: () => void) => { + queueMicrotask(callback); + return 1; + }, + }) as Promise); + + const observation = { + step: CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, + outcome: "denied", + statusCode: null, + errorCode: "PREMATURE_CLOSE", + }; + expect(stderr).toBe(""); + expect(childProcess.exitCode).toBeUndefined(); + expect(JSON.parse(writes.get(CREDENTIAL_WINDOW_PATHS.acknowledgement) ?? "")).toEqual( + observation, + ); + expect(JSON.parse(stdout)).toEqual({ revision: "v7", outcomes: [observation] }); + }); + + it("accepts bounded request diagnostics and rejects raw error text (#10155)", () => { + expect( + parseCredentialWindowRequestObservation( + JSON.stringify({ + step: CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, + outcome: "allowed", + statusCode: 200, + errorCode: null, + }), + ), + ).toEqual({ + step: CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, + outcome: "allowed", + statusCode: 200, + errorCode: null, + }); + expect( + parseCredentialWindowRequestObservation( + JSON.stringify({ + step: CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, + outcome: "denied", + statusCode: 503, + errorCode: null, + }), + ), + ).toEqual({ + step: CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, + outcome: "denied", + statusCode: 503, + errorCode: null, + }); + expect( + parseCredentialWindowRequestObservation( + JSON.stringify({ + step: CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, + outcome: "denied", + statusCode: null, + errorCode: "ECONNRESET", + }), + ), + ).toEqual({ + step: CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, + outcome: "denied", + statusCode: null, + errorCode: "ECONNRESET", + }); + expect( + parseCredentialWindowRequestObservation( + JSON.stringify({ + step: CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, + outcome: "denied", + statusCode: null, + errorCode: "Error: fixture credential leaked", + }), + ), + ).toBeNull(); + expect( + parseCredentialWindowRequestObservation( + JSON.stringify({ + step: CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, + outcome: "allowed", + statusCode: 401, + errorCode: null, + }), + ), + ).toBeNull(); + }); + it("builds explicit bounded expiry and attached-key-removal updates", () => { expect(CREDENTIAL_WINDOW_EXPIRY_DELAY_MS).toBe(3 * 60_000); expect(buildCredentialWindowProviderUpdateArgs("owned-provider", 123_456)).toEqual([ @@ -104,7 +263,9 @@ describe("OpenShell exact-main credential generation-window proof", () => { expect(liveTarget).toContain('["nemoclaw-start", "node", "-e"'); expect(liveTarget).toContain("CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry"); expect(liveTarget).toContain("CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval"); - expect(liveTarget).toContain('["sandbox", "provider", "detach"'); + expect(liveTarget).toContain('[SANDBOX_NAME, "mcp", "remove", SERVER_NAME]'); + expect(liveTarget).toMatch(/\[\s*SANDBOX_NAME,\s*"mcp",\s*"add",\s*SERVER_NAME,/u); + expect(liveTarget).not.toContain('["sandbox", "provider", "detach"'); expect(liveTarget).toContain('[SANDBOX_NAME, "mcp", "restart", SERVER_NAME]'); expect(liveTarget).toContain('[SANDBOX_NAME, "rebuild", "--yes"]'); expect(liveTarget).toContain('!request.auth.includes("openshell:resolve:env")'); diff --git a/test/e2e/README.md b/test/e2e/README.md index 5ee9f9568eb..60802c7eec2 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -693,10 +693,11 @@ lanes: The OpenClaw shards of the matrix jobs, the `openclaw` MCP shard, `mcp-bridge-dev`, and `openshell-credential-generation-window` remain on `ubuntu-latest`; unrelated jobs retain their existing runner assignments. -The credential-generation window runs as an independent fresh-runner job in -parallel with the stable MCP agent matrix. Empty-selector dispatches and -explicit `mcp-bridge` selections run both jobs, while the credential-window job -keeps its own exact-release provenance, secret scan, and artifact. +The credential-generation window runs as an independent fresh-runner job after +the trusted planner and applicable base-image publication check, in parallel +with the stable MCP agent matrix. Empty-selector dispatches and explicit +`mcp-bridge` selections run both jobs, while the credential-window job keeps +its own exact-release provenance, secret scan, and artifact. Before setting the variable, an organization owner must: 1. Create a GitHub-hosted Ubuntu x64 larger runner with 8 vCPU, 32 GB RAM, and @@ -1274,6 +1275,7 @@ Selected jobs retain their runner, credential, evidence, and cleanup boundaries. A main push can queue repository-owned GPU runners or create external resources when a selected target requires them. The main-run observer records attempt evidence but does not request broad failed-job reruns. Each E2E test owns any bounded operation-level retry policy. +[`RETRY_INVENTORY.md`](RETRY_INVENTORY.md) records each policy's owner and retry boundary. `Exact staging Brev Launchable` runs only for a trusted manual dispatch against `main`. The job reads these credentials from repository Actions secrets: diff --git a/test/e2e/RETRY_INVENTORY.md b/test/e2e/RETRY_INVENTORY.md new file mode 100644 index 00000000000..691884b9fc1 --- /dev/null +++ b/test/e2e/RETRY_INVENTORY.md @@ -0,0 +1,10 @@ + + + +# E2E Retry Inventory + +This inventory records bounded operation retries in live E2E tests. + +| Operation | Owner | Retry condition | Limit | Idempotence and reconciliation | Evidence | +|---|---|---|---:|---|---| +| OpenClaw MCP tool discovery | `mcp-bridge-live-e2e` | `mcp status --tools --json` reports `MCP tool discovery request failed`, and the fixture received no request | 2 attempts | The status operation is read-only. Before attempt 2, `mcp restart fake` reconciles the committed provider and policy without a host credential. | `openclaw-mcp-tool-discovery-diagnostics.json` records the failure class, reconciliation result, and each attempt. | diff --git a/test/e2e/live/openshell-credential-generation-window.ts b/test/e2e/live/mcp-bridge-credential-generation-window.ts similarity index 67% rename from test/e2e/live/openshell-credential-generation-window.ts rename to test/e2e/live/mcp-bridge-credential-generation-window.ts index 5d56d44993c..4e1493935a0 100644 --- a/test/e2e/live/openshell-credential-generation-window.ts +++ b/test/e2e/live/mcp-bridge-credential-generation-window.ts @@ -24,7 +24,7 @@ export const CREDENTIAL_WINDOW_STEPS = { fallbackAfterEviction: "fallback-after-eviction", deniedAfterKeyRemoval: "denied-after-key-removal", deniedAfterDetach: "denied-after-detach", - fallbackAfterRestart: "fallback-after-restart", + deniedAfterReadd: "denied-after-readd", stop: "stop", } as const; @@ -34,7 +34,68 @@ export type CredentialWindowRequestStep = | (typeof CREDENTIAL_WINDOW_STEPS)["fallbackAfterEviction"] | (typeof CREDENTIAL_WINDOW_STEPS)["deniedAfterKeyRemoval"] | (typeof CREDENTIAL_WINDOW_STEPS)["deniedAfterDetach"] - | (typeof CREDENTIAL_WINDOW_STEPS)["fallbackAfterRestart"]; + | (typeof CREDENTIAL_WINDOW_STEPS)["deniedAfterReadd"]; + +export interface CredentialWindowRequestObservation { + readonly step: CredentialWindowRequestStep; + readonly outcome: "allowed" | "denied"; + readonly statusCode: number | null; + readonly errorCode: string | null; +} + +const CREDENTIAL_WINDOW_REQUEST_STEPS: readonly CredentialWindowRequestStep[] = [ + CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, + CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, + CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, + CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, + CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, + CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, +]; + +export function parseCredentialWindowRequestObservation( + output: string, +): CredentialWindowRequestObservation | null { + try { + const parsed = JSON.parse(output.trim()) as Record; + if ( + !parsed || + typeof parsed !== "object" || + Array.isArray(parsed) || + Object.keys(parsed).sort().join(",") !== "errorCode,outcome,statusCode,step" || + typeof parsed.step !== "string" || + !CREDENTIAL_WINDOW_REQUEST_STEPS.includes(parsed.step as CredentialWindowRequestStep) || + (parsed.outcome !== "allowed" && parsed.outcome !== "denied") + ) { + return null; + } + + const statusCode = parsed.statusCode; + const errorCode = parsed.errorCode; + const validStatusCode = + statusCode === null || + (Number.isInteger(statusCode) && + (statusCode as number) >= 100 && + (statusCode as number) <= 599); + const validErrorCode = + errorCode === null || (typeof errorCode === "string" && /^[A-Z0-9_]{1,64}$/u.test(errorCode)); + if (!validStatusCode || !validErrorCode) return null; + + if (parsed.outcome === "allowed") { + if (statusCode !== 200 || errorCode !== null) return null; + } else if (statusCode === 200 || (statusCode === null) === (errorCode === null)) { + return null; + } + + return { + step: parsed.step as CredentialWindowRequestStep, + outcome: parsed.outcome, + statusCode: statusCode as number | null, + errorCode: errorCode as string | null, + }; + } catch { + return null; + } +} export function credentialWindowSecret(generation: number): string { return `${MCP_BRIDGE_TEST_CREDENTIALS.generationWindow}${String(generation).padStart(2, "0")}`; @@ -108,15 +169,25 @@ const requestSteps = new Set([ config.steps.fallbackAfterEviction, config.steps.deniedAfterKeyRemoval, config.steps.deniedAfterDetach, - config.steps.fallbackAfterRestart, + config.steps.deniedAfterReadd, ]); const seen = new Set(); const outcomes = []; const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); +const normalizeErrorCode = (error) => { + const code = error && typeof error.code === "string" ? error.code.toUpperCase() : "UNKNOWN"; + return /^[A-Z0-9_]{1,64}$/.test(code) ? code : "UNKNOWN"; +}; const writePrivateJson = (file, value) => { fs.writeFileSync(file, JSON.stringify(value) + "\\n", { encoding: "utf8", mode: 0o600 }); }; const request = (step) => new Promise((resolve) => { + let settled = false; + const finish = (observation) => { + if (settled) return; + settled = true; + resolve({ step, ...observation }); + }; const body = JSON.stringify({ jsonrpc: "2.0", id: config.requestPrefix + ":" + step, @@ -134,10 +205,29 @@ const request = (step) => new Promise((resolve) => { }, }, (response) => { response.resume(); - response.on("end", () => resolve(response.statusCode === 200 ? "allowed" : "denied")); + response.on("end", () => { + const statusCode = Number.isInteger(response.statusCode) ? response.statusCode : null; + finish({ + outcome: statusCode === 200 ? "allowed" : "denied", + statusCode, + errorCode: statusCode === null ? "MISSING_STATUS" : null, + }); + }); + response.on("close", () => { + if (!response.readableEnded) { + finish({ outcome: "denied", statusCode: null, errorCode: "PREMATURE_CLOSE" }); + } + }); + }); + outbound.on("error", (error) => finish({ + outcome: "denied", + statusCode: null, + errorCode: normalizeErrorCode(error), + })); + outbound.setTimeout(30_000, () => { + finish({ outcome: "denied", statusCode: null, errorCode: "TIMEOUT" }); + outbound.destroy(); }); - outbound.on("error", () => resolve("denied")); - outbound.setTimeout(30_000, () => outbound.destroy()); outbound.end(body); }); @@ -153,9 +243,9 @@ const deadline = Date.now() + config.maxRuntimeMs; stopped = true; } else if (requestSteps.has(step) && !seen.has(step)) { seen.add(step); - const outcome = await request(step); - outcomes.push({ step, outcome }); - writePrivateJson(config.acknowledgementPath, { step, outcome }); + const observation = await request(step); + outcomes.push(observation); + writePrivateJson(config.acknowledgementPath, observation); } if (!stopped) await delay(200); } diff --git a/test/e2e/live/mcp-bridge-tool-discovery.ts b/test/e2e/live/mcp-bridge-tool-discovery.ts index b3bf15fc6e2..28a4c6f8848 100644 --- a/test/e2e/live/mcp-bridge-tool-discovery.ts +++ b/test/e2e/live/mcp-bridge-tool-discovery.ts @@ -8,6 +8,7 @@ import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { assertExitZero } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { TestProgress } from "../fixtures/progress.ts"; +import { runBoundedRetry, type RetryEvidence } from "../fixtures/retry-policy.ts"; import type { FakeMcpHttpsServer, FakeMcpRequest } from "./mcp-bridge-servers.ts"; export interface AuthenticatedMcpDiscoveryTarget { @@ -75,6 +76,7 @@ function buildMcpToolDiscoveryDiagnostics( status: McpToolDiscoveryStatusJson, requests: readonly FakeMcpRequest[], expectedSecret: string, + retry: RetryEvidence, ): Record { return { provider: { @@ -105,6 +107,7 @@ function buildMcpToolDiscoveryDiagnostics( truncated: status.toolDiscovery.truncated, ...(status.toolDiscovery.detail !== undefined ? { detail: status.toolDiscovery.detail } : {}), }, + retry, requests: requests.map((request) => ({ httpMethod: request.method, rpcMethod: request.rpcMethod ?? null, @@ -434,49 +437,94 @@ export async function assertAuthenticatedMcpToolDiscovery( credentialKey?: string; hostSecret: string; progress: Pick; + reconcileTransportFailure?: () => Promise; serverName?: string; }, + deps: { + sleep: (milliseconds: number) => Promise; + } = { + sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), + }, ): Promise { const credentialKey = options.credentialKey ?? "FAKE_MCP_SECRET"; const serverName = options.serverName ?? "fake"; + const reconcileTransportFailure = options.reconcileTransportFailure; const requestOffset = fakeMcp.requests.length; - let status: Awaited> | undefined; - let statusJson: McpToolDiscoveryStatusJson | undefined; - for (let attempt = 1; attempt <= MCP_TOOL_DISCOVERY_ATTEMPTS; attempt += 1) { - status = await host.nemoclaw( - [options.sandboxName, "mcp", "status", serverName, "--tools", "--json"], - { - artifactName: `${options.artifactPrefix}-mcp-status-tools-json${attempt === 1 ? "" : `-retry-${attempt}`}`, - env: { - ...buildAvailabilityProbeEnv(), - [credentialKey]: options.hostSecret, + let reconciliationError: unknown; + const retryResult = await runBoundedRetry({ + operation: "mcp-tool-discovery", + owner: "mcp-bridge-live-e2e", + idempotence: reconcileTransportFailure ? "reconciled-mutation" : "read-only", + maxAttempts: MCP_TOOL_DISCOVERY_ATTEMPTS, + run: async (attempt) => { + const status = await host.nemoclaw( + [options.sandboxName, "mcp", "status", serverName, "--tools", "--json"], + { + artifactName: `${options.artifactPrefix}-mcp-status-tools-json${attempt === 1 ? "" : `-retry-${attempt}`}`, + env: { + ...buildAvailabilityProbeEnv(), + [credentialKey]: options.hostSecret, + }, + redactionValues: [options.hostSecret], + timeoutMs: 60_000, }, - redactionValues: [options.hostSecret], - timeoutMs: 60_000, - }, - ); - assertExitZero(status, `${options.artifactPrefix} mcp status --tools --json`); - statusJson = JSON.parse(status.stdout) as McpToolDiscoveryStatusJson; - if ( - !shouldRetryMcpToolDiscoveryTransportFailure( - statusJson.toolDiscovery, + ); + assertExitZero(status, `${options.artifactPrefix} mcp status --tools --json`); + return { + status, + statusJson: JSON.parse(status.stdout) as McpToolDiscoveryStatusJson, + }; + }, + classify: (value, error) => { + if (error || !value) return { outcome: "failed", failureClass: "deterministic" }; + if (value.statusJson.toolDiscovery.ok) return { outcome: "passed" }; + return shouldRetryMcpToolDiscoveryTransportFailure( + value.statusJson.toolDiscovery, fakeMcp.requests.slice(requestOffset), - attempt, + 1, ) - ) { - break; - } - options.progress.event( - "MCP tool discovery transport failed before reaching the fixture; retrying once", - ); - await new Promise((resolve) => setTimeout(resolve, MCP_TOOL_DISCOVERY_RETRY_DELAY_MS)); + ? { outcome: "failed", failureClass: "transient-external" } + : { outcome: "failed", failureClass: "deterministic" }; + }, + reconcile: reconcileTransportFailure + ? async () => { + options.progress.event( + "MCP tool discovery transport failed before reaching the fixture; restarting the bridge once", + ); + try { + await reconcileTransportFailure(); + return true; + } catch (error) { + reconciliationError = error; + return false; + } + } + : undefined, + delayMs: MCP_TOOL_DISCOVERY_RETRY_DELAY_MS, + sleep: async (milliseconds) => { + if (!reconcileTransportFailure) { + options.progress.event( + "MCP tool discovery transport failed before reaching the fixture; retrying once", + ); + } + await deps.sleep(milliseconds); + }, + }); + if (!retryResult.value) { + throw new Error("MCP tool discovery did not return status"); } - if (!status || !statusJson) throw new Error("MCP tool discovery did not run"); + const { status, statusJson } = retryResult.value; const discoveryRequests = fakeMcp.requests.slice(requestOffset); await options.artifacts.writeJson( `${options.artifactPrefix}-mcp-tool-discovery-diagnostics.json`, - buildMcpToolDiscoveryDiagnostics(statusJson, discoveryRequests, options.hostSecret), + buildMcpToolDiscoveryDiagnostics( + statusJson, + discoveryRequests, + options.hostSecret, + retryResult.evidence, + ), ); + if (reconciliationError) throw reconciliationError; expect(statusJson.provider.credentialResolution).toBeUndefined(); expect(statusJson.toolDiscovery).toMatchObject({ ok: true, diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index b3a607a35ba..613fc04fa15 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -809,6 +809,12 @@ test("mcp-bridge", { artifactPrefix: "openclaw", hostSecret: HOST_SECRET, progress, + reconcileTransportFailure: () => + restartBridgeWithoutHostSecret( + host, + OPENCLAW_SANDBOX_NAME, + "openclaw-tool-discovery-retry", + ), }); await assertBridgeInfrastructure(host, sandbox, { sandboxName: OPENCLAW_SANDBOX_NAME, diff --git a/test/e2e/live/openshell-credential-generation-window.test.ts b/test/e2e/live/openshell-credential-generation-window.test.ts index bbc9b811aae..da5ca793d4c 100644 --- a/test/e2e/live/openshell-credential-generation-window.test.ts +++ b/test/e2e/live/openshell-credential-generation-window.test.ts @@ -30,11 +30,13 @@ import { CREDENTIAL_WINDOW_REQUEST_PREFIX, CREDENTIAL_WINDOW_ROTATION_COUNT, CREDENTIAL_WINDOW_STEPS, + type CredentialWindowRequestObservation, type CredentialWindowRequestStep, credentialWindowRequestId, credentialWindowSecrets, OPENSHELL_RETAINED_CREDENTIAL_GENERATIONS, -} from "./openshell-credential-generation-window.ts"; + parseCredentialWindowRequestObservation, +} from "./mcp-bridge-credential-generation-window.ts"; const SANDBOX_NAME = "e2e-cred-window"; const SERVER_NAME = "fake"; @@ -50,7 +52,7 @@ interface CredentialWindowRequest { interface CredentialWindowChildResult { readonly revision: string; - readonly outcomes: Array<{ step: string; outcome: string }>; + readonly outcomes: CredentialWindowRequestObservation[]; } function openshellEnv(): NodeJS.ProcessEnv { @@ -231,9 +233,8 @@ async function waitForReadyRevision(sandbox: SandboxClient): Promise { async function waitForAcknowledgement( sandbox: SandboxClient, step: CredentialWindowRequestStep, - outcome: "allowed" | "denied", -): Promise { - const expected = JSON.stringify({ step, outcome }); +): Promise { + let matchingOutput = ""; await expect .poll( async () => { @@ -242,7 +243,14 @@ async function waitForAcknowledgement( CREDENTIAL_WINDOW_PATHS.acknowledgement, `credential-window-${step}-ack-poll`, ); - return result.exitCode === 0 ? result.stdout.trim() : ""; + try { + const output = result.exitCode === 0 ? result.stdout.trim() : "{}"; + const parsed = JSON.parse(output) as { step?: unknown }; + matchingOutput = parsed.step === step ? result.stdout : matchingOutput; + return parsed.step; + } catch { + return ""; + } }, { interval: 500, @@ -250,7 +258,10 @@ async function waitForAcknowledgement( message: `old child acknowledgement for ${step}`, }, ) - .toBe(expected); + .toBe(step); + const observation = parseCredentialWindowRequestObservation(matchingOutput); + expect(observation, `old child acknowledgement for ${step} must be bounded`).not.toBeNull(); + return observation!; } async function rotateCredential( @@ -329,8 +340,8 @@ test("openshell-credential-generation-window", { "attach the MCP provider and observe its initial generation", "prove a retained credential generation expires", "rotate beyond the retained generation window", - "prove key removal and provider detach revoke access", - "restart the bridge and confirm old-process fallback", + "prove key and bridge removal revoke access", + "re-add the bridge and keep the old process revoked", "rebuild the sandbox and confirm credential reuse", "remove the MCP bridge and audit denied requests", ], @@ -345,13 +356,13 @@ test("openshell-credential-generation-window", { const initialSecret = allSecrets[0]!; const rotationSecrets = allSecrets.slice(1, CREDENTIAL_WINDOW_ROTATION_COUNT + 1); const expirySecret = allSecrets.at(-2)!; - const restartSecret = allSecrets.at(-1)!; + const readdSecret = allSecrets.at(-1)!; artifacts.addRedactionValues([COMPATIBLE_KEY, ...allSecrets]); await artifacts.target.declare({ id: "openshell-credential-generation-window", contracts: [ "OpenShell f27ff150 retained credential generations", - "NemoClaw MCP detach, restart, and rebuild lifecycle", + "NemoClaw MCP remove, re-add, and rebuild lifecycle", ], sourceRevision: "3dee5570a46076a57a3b056f35f35ebc0861ac85", }); @@ -433,8 +444,9 @@ test("openshell-credential-generation-window", { timeoutMs: 60_000, }); expectExitZero(status, "inspect credential-window MCP bridge"); - const providerName = (JSON.parse(status.stdout) as { provider: { name: string } }).provider.name; - expect(providerName).toMatch(/^e2e-cred-window-mcp-fake-[a-f0-9]{16}$/u); + const initialProviderName = (JSON.parse(status.stdout) as { provider: { name: string } }).provider + .name; + expect(initialProviderName).toMatch(/^e2e-cred-window-mcp-fake-[a-f0-9]{16}$/u); const originalRevision = await observeFreshRevision( sandbox, @@ -462,7 +474,7 @@ test("openshell-credential-generation-window", { fakeMcp.setSecret(expirySecret); await updateProviderCredential( sandbox, - providerName, + initialProviderName, expirySecret, expiryAtMs, allSecrets, @@ -495,7 +507,14 @@ test("openshell-credential-generation-window", { CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, "credential-window-signal-before-expiry", ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, "allowed"); + expect( + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry), + ).toEqual({ + step: CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, + outcome: "allowed", + statusCode: 200, + errorCode: null, + }); expect( requestEvidence( fakeMcp, @@ -511,7 +530,7 @@ test("openshell-credential-generation-window", { fakeMcp.setSecret(initialSecret); await updateProviderCredential( sandbox, - providerName, + initialProviderName, initialSecret, 0, allSecrets, @@ -552,7 +571,12 @@ test("openshell-credential-generation-window", { CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, "credential-window-signal-after-expiry", ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, "denied"); + expect( + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry), + ).toMatchObject({ + step: CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, + outcome: "denied", + }); expect( requestEvidence( fakeMcp, @@ -576,7 +600,7 @@ test("openshell-credential-generation-window", { expect(expiryChildResult).toBeDefined(); expectExitZero(expiryChildResult!, "retained-expiry credential-window child"); - expect(parseLastJsonLine(expiryChildResult!.stdout)).toEqual({ + expect(parseLastJsonLine(expiryChildResult!.stdout)).toMatchObject({ revision: expiryChildRevision, outcomes: [ { @@ -620,7 +644,8 @@ test("openshell-credential-generation-window", { ); let oldChildRevision = ""; let oldChildResult: ShellProbeResult | undefined; - let restartedRevision = ""; + let readdedProviderName = ""; + let readdedRevision = ""; const observedRevisions = [restoredRevision]; try { oldChildRevision = await waitForReadyRevision(sandbox); @@ -650,7 +675,12 @@ test("openshell-credential-generation-window", { CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, "credential-window-signal-fallback-after-eviction", ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, "denied"); + expect( + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction), + ).toMatchObject({ + step: CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, + outcome: "denied", + }); expect( requestEvidence( fakeMcp, @@ -677,10 +707,10 @@ test("openshell-credential-generation-window", { placeholderAbsent: true, }); - progress.phase("prove key removal and provider detach revoke access"); + progress.phase("prove key and bridge removal revoke access"); await updateProviderCredential( sandbox, - providerName, + initialProviderName, "", 0, allSecrets, @@ -695,7 +725,12 @@ test("openshell-credential-generation-window", { CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, "credential-window-signal-after-key-removal", ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, "denied"); + expect( + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval), + ).toMatchObject({ + step: CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, + outcome: "denied", + }); expect( requestEvidence( fakeMcp, @@ -704,14 +739,14 @@ test("openshell-credential-generation-window", { ).seen, ).toBe(false); - fakeMcp.setSecret(restartSecret); + fakeMcp.setSecret(readdSecret); await updateProviderCredential( sandbox, - providerName, - restartSecret, + initialProviderName, + readdSecret, 0, allSecrets, - "credential-window-restore-current-key-before-detach", + "credential-window-restore-current-key-before-removal", ); const restoredKeyRevision = await observeDistinctFreshRevision( sandbox, @@ -730,22 +765,21 @@ test("openshell-credential-generation-window", { revision: restoredKeyRevision, status: 200, }); - expect(requestEvidence(fakeMcp, freshAfterKeyRestoreId, restartSecret)).toEqual({ + expect(requestEvidence(fakeMcp, freshAfterKeyRestoreId, readdSecret)).toEqual({ seen: true, credentialRewritten: true, placeholderAbsent: true, }); - const detach = await sandbox.openshell( - ["sandbox", "provider", "detach", SANDBOX_NAME, providerName], + const removeBeforeReadd = await host.nemoclaw( + [SANDBOX_NAME, "mcp", "remove", SERVER_NAME], { - artifactName: "credential-window-direct-provider-detach", - env: openshellEnv(), - timeoutMs: 90_000, + artifactName: "credential-window-remove-before-readd", + env: buildAvailabilityProbeEnv(), + timeoutMs: 4 * 60_000, }, ); - expectExitZero(detach, "detach credential-window provider"); - expect(resultText(detach)).toMatch(/Detached provider/iu); + expectExitZero(removeBeforeReadd, "remove credential-window bridge before re-add"); await expectFreshCredentialAbsent( sandbox, "credential-window-fresh-credential-absent-after-detach", @@ -756,54 +790,99 @@ test("openshell-credential-generation-window", { CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, "credential-window-signal-after-detach", ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, "denied"); + expect( + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterDetach), + ).toMatchObject({ + step: CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, + outcome: "denied", + }); expect( requestEvidence( fakeMcp, credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterDetach), - restartSecret, + readdSecret, ).seen, ).toBe(false); - progress.phase("restart the bridge and confirm old-process fallback"); - await rotateCredential( - host, - fakeMcp, - restartSecret, - CREDENTIAL_WINDOW_ROTATION_COUNT + 1, - allSecrets, + progress.phase("re-add the bridge and keep the old process revoked"); + fakeMcp.setSecret(readdSecret); + const readd = await host.nemoclaw( + [ + SANDBOX_NAME, + "mcp", + "add", + SERVER_NAME, + "--url", + tunnel.url, + "--env", + CREDENTIAL_WINDOW_ENV_NAME, + ], + { + artifactName: "credential-window-readd-after-removal", + env: { + ...buildAvailabilityProbeEnv(), + [CREDENTIAL_WINDOW_ENV_NAME]: readdSecret, + }, + redactionValues: [...allSecrets], + timeoutMs: 4 * 60_000, + }, ); - restartedRevision = await observeFreshRevision( + expectExitZero(readd, "re-add credential-window bridge"); + const readdedStatus = await host.nemoclaw( + [SANDBOX_NAME, "mcp", "status", SERVER_NAME, "--json"], + { + artifactName: "credential-window-mcp-status-after-readd", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + expectExitZero(readdedStatus, "inspect re-added credential-window MCP bridge"); + readdedProviderName = ( + JSON.parse(readdedStatus.stdout) as { provider: { name: string } } + ).provider.name; + expect(readdedProviderName).toMatch(/^e2e-cred-window-mcp-fake-[a-f0-9]{16}$/u); + expect(readdedProviderName).not.toBe(initialProviderName); + readdedRevision = await observeFreshRevision( sandbox, - "credential-window-fresh-revision-after-restart", + "credential-window-fresh-revision-after-readd", ); - expect(restartedRevision).not.toBe(currentRevision); - expect(restartedRevision).not.toBe(restoredKeyRevision); + expect(readdedRevision).not.toBe(currentRevision); + expect(readdedRevision).not.toBe(restoredKeyRevision); + const freshAfterReaddId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-readd`; + const freshAfterReadd = await runFreshRequest( + sandbox, + tunnel.url, + freshAfterReaddId, + allSecrets, + "credential-window-fresh-request-after-readd", + ); + expect(freshAfterReadd).toEqual({ + revision: readdedRevision, + status: 200, + }); + expect(requestEvidence(fakeMcp, freshAfterReaddId, readdSecret)).toEqual({ + seen: true, + credentialRewritten: true, + placeholderAbsent: true, + }); await writeControl( sandbox, - CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart, - "credential-window-signal-fallback-after-restart", + CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, + "credential-window-signal-denied-after-readd", ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart, "allowed"); - await expect - .poll( - () => - requestEvidence( - fakeMcp, - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart), - restartSecret, - ), - { - interval: 500, - timeout: 30_000, - message: "old revision fallback after restart", - }, - ) - .toEqual({ - seen: true, - credentialRewritten: true, - placeholderAbsent: true, - }); + expect( + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterReadd), + ).toMatchObject({ + step: CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, + outcome: "denied", + }); + expect( + requestEvidence( + fakeMcp, + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd), + readdSecret, + ).seen, + ).toBe(false); } finally { await writeControl( sandbox, @@ -821,7 +900,7 @@ test("openshell-credential-generation-window", { expect(oldChildResult).toBeDefined(); expectExitZero(oldChildResult!, "old credential-window child"); const childSummary = parseLastJsonLine(oldChildResult!.stdout); - expect(childSummary).toEqual({ + expect(childSummary).toMatchObject({ revision: oldChildRevision, outcomes: [ { @@ -831,8 +910,8 @@ test("openshell-credential-generation-window", { { step: CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, outcome: "denied" }, { step: CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, outcome: "denied" }, { - step: CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart, - outcome: "allowed", + step: CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, + outcome: "denied", }, ], }); @@ -865,7 +944,7 @@ test("openshell-credential-generation-window", { revision: rebuiltRevision, status: 200, }); - expect(requestEvidence(fakeMcp, freshAfterRebuildId, restartSecret)).toEqual({ + expect(requestEvidence(fakeMcp, freshAfterRebuildId, readdSecret)).toEqual({ seen: true, credentialRewritten: true, placeholderAbsent: true, @@ -884,7 +963,7 @@ test("openshell-credential-generation-window", { ); const providerAfterRemove = await host.command( host.openshellCommandPath, - ["provider", "get", providerName], + ["provider", "get", readdedProviderName], { artifactName: "credential-window-provider-absent-after-remove", env: openshellEnv(), @@ -906,6 +985,9 @@ test("openshell-credential-generation-window", { expect(upstreamRequestIds).not.toContain( credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterDetach), ); + expect(upstreamRequestIds).not.toContain( + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd), + ); expect( fakeMcp.requests.every( (request: CredentialWindowRequest) => !request.auth.includes("openshell:resolve:env"), @@ -915,8 +997,8 @@ test("openshell-credential-generation-window", { id: "openshell-credential-generation-window", expiryRevision, oldChildRevision, + readdedRevision, rebuiltRevision, - restartedRevision, rotations: CREDENTIAL_WINDOW_ROTATION_COUNT, }); }); diff --git a/test/e2e/support/cli-artifact-workflow-boundary.test.ts b/test/e2e/support/cli-artifact-workflow-boundary.test.ts index 531fce5ec1a..f32a1e5c3b4 100644 --- a/test/e2e/support/cli-artifact-workflow-boundary.test.ts +++ b/test/e2e/support/cli-artifact-workflow-boundary.test.ts @@ -440,6 +440,15 @@ describe("exact-commit CLI artifact workflow boundary", () => { expect(validateCliArtifactWorkflowBoundary(readWorkflow())).toEqual([]); }); + it("keeps the credential-window image prerequisite alongside the CLI producer (#10155)", () => { + const workflow = workflowFixture(); + workflow.jobs["openshell-credential-generation-window"].needs = "base-image-publication"; + + expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( + "openshell-credential-generation-window must depend directly on the CLI artifact producer", + ); + }); + it("rejects dependency caching before trusted installation (#9051)", () => { const workflow = workflowFixture(); const setupNode = requireStep( diff --git a/test/e2e/support/mcp-bridge-tool-discovery.test.ts b/test/e2e/support/mcp-bridge-tool-discovery.test.ts index 85841251b8a..4519015c670 100644 --- a/test/e2e/support/mcp-bridge-tool-discovery.test.ts +++ b/test/e2e/support/mcp-bridge-tool-discovery.test.ts @@ -321,6 +321,7 @@ describe("authenticated MCP tool discovery transport retry", () => { const artifactRoot = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-mcp-diagnostics-")); artifactRoots.push(artifactRoot); const artifacts = new ArtifactSink(artifactRoot); + const reconcileTransportFailure = vi.fn().mockResolvedValue(undefined); await expect( assertAuthenticatedMcpToolDiscovery(host, fakeMcp, { @@ -329,6 +330,7 @@ describe("authenticated MCP tool discovery transport retry", () => { artifactPrefix: "openclaw-trusted-private", hostSecret: EXPECTED_SECRET, progress: { event: vi.fn() }, + reconcileTransportFailure, }), ).rejects.toThrow(); @@ -355,6 +357,22 @@ describe("authenticated MCP tool discovery transport retry", () => { truncated: false, detail: "MCP tool discovery request failed", }, + retry: { + schemaVersion: 1, + operation: "mcp-tool-discovery", + owner: "mcp-bridge-live-e2e", + idempotence: "reconciled-mutation", + maxAttempts: 2, + outcome: "failed-no-retry", + attempts: [ + { + attempt: 1, + outcome: "failed", + failureClass: "deterministic", + retryScheduled: false, + }, + ], + }, requests: [ { httpMethod: "POST", @@ -381,6 +399,99 @@ describe("authenticated MCP tool discovery transport retry", () => { expect(diagnostics).not.toContain(EXPECTED_SECRET); expect(diagnostics).not.toContain(SESSION_ID); expect(diagnostics).not.toContain(PROTOCOL_VERSION); + expect(reconcileTransportFailure).not.toHaveBeenCalled(); + }); + + it("restarts the bridge once before retrying a no-request transport failure", async () => { + const failedStatus = { + provider: { + registryPresent: true, + gatewayPresent: true, + attached: true, + credentialReady: true, + }, + policy: { registryPresent: true, gatewayPresent: true }, + adapter: { registered: true }, + toolDiscovery: { + ok: false, + count: 0, + tools: [], + truncated: false, + detail: "MCP tool discovery request failed", + }, + }; + const passedStatus = { + ...failedStatus, + toolDiscovery: { + ok: true, + count: 2, + tools: ["fake_echo", "fake_status"], + truncated: false, + }, + }; + const fakeMcp = fakeDiscoveryServer(); + const host = { + nemoclaw: vi + .fn() + .mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify(failedStatus), stderr: "" }) + .mockImplementationOnce(async () => { + fakeMcp.requests.push( + successfulInitialize(), + request("notifications/initialized"), + request("tools/list"), + request("tools/list"), + ); + return { exitCode: 0, stdout: JSON.stringify(passedStatus), stderr: "" }; + }), + } as unknown as Parameters[0]; + const artifacts = discoveryArtifacts(); + const progress = { event: vi.fn() }; + const reconcileTransportFailure = vi.fn().mockResolvedValue(undefined); + const sleep = vi.fn().mockResolvedValue(undefined); + + await assertAuthenticatedMcpToolDiscovery( + host, + fakeMcp, + { + artifacts, + sandboxName: "sandbox", + artifactPrefix: "openclaw", + hostSecret: EXPECTED_SECRET, + progress, + reconcileTransportFailure, + }, + { sleep }, + ); + + expect(host.nemoclaw).toHaveBeenCalledTimes(2); + expect(reconcileTransportFailure).toHaveBeenCalledTimes(1); + expect(sleep).toHaveBeenCalledWith(1_000); + expect(progress.event).toHaveBeenCalledWith( + "MCP tool discovery transport failed before reaching the fixture; restarting the bridge once", + ); + expect(artifacts.writeJson).toHaveBeenCalledWith( + "openclaw-mcp-tool-discovery-diagnostics.json", + expect.objectContaining({ + retry: { + schemaVersion: 1, + operation: "mcp-tool-discovery", + owner: "mcp-bridge-live-e2e", + idempotence: "reconciled-mutation", + maxAttempts: 2, + outcome: "passed-after-retry", + attempts: [ + { + attempt: 1, + outcome: "failed", + failureClass: "transient-external", + reconciled: true, + retryScheduled: true, + }, + { attempt: 2, outcome: "passed", retryScheduled: false }, + ], + }, + }), + ); }); it("retries one generic transport failure before any request reaches the fixture", () => { diff --git a/test/e2e/support/mcp-workflow-boundary.test.ts b/test/e2e/support/mcp-workflow-boundary.test.ts index cc1bcfc2ae0..d0df0eda5db 100644 --- a/test/e2e/support/mcp-workflow-boundary.test.ts +++ b/test/e2e/support/mcp-workflow-boundary.test.ts @@ -101,7 +101,7 @@ describe("MCP workflow artifact boundary", () => { expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toEqual( expect.arrayContaining([ "mcp-bridge must not serialize the independent credential generation-window proof", - "openshell-credential-generation-window must depend only on matrix generation so it can run in parallel", + "openshell-credential-generation-window must depend on its trusted image and matrix producers without MCP serialization", ]), ); } finally { diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 9f3e980ee16..1376a22aca1 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -34,9 +34,7 @@ import { writeE2eWorkflowPlanCiOutput, } from "../../../tools/e2e/workflow-plan.mts"; import { REPO_ROOT } from "../fixtures/paths.ts"; -import { listTargets } from "../registry/registry.ts"; import { buildLiveTargetMatrix } from "../registry/run.ts"; -import { liveTargetSupport } from "../registry/runtime-support.ts"; const PLANNER_CLI = path.join(REPO_ROOT, "tools", "e2e", "workflow-plan.mts"); const TSX = path.join(REPO_ROOT, "node_modules", ".bin", "tsx"); @@ -71,6 +69,21 @@ function expectedCiOutput(plan: ReturnType): string ].join("\n"); } +function captureWorkflowPlanCli(args: string[]): string { + let output = ""; + const write = process.stdout.write; + process.stdout.write = ((chunk: string | Uint8Array) => { + output += chunk.toString(); + return true; + }) as typeof process.stdout.write; + try { + runE2eWorkflowPlanCli(args); + return output; + } finally { + process.stdout.write = write; + } +} + function prCandidatePlan( plan: ReturnType, ): ReturnType { @@ -1201,17 +1214,7 @@ describe("E2E workflow plan", () => { }); it("emits one compact JSON line with the deterministic workflow-output schema", () => { - let output = ""; - const write = process.stdout.write; - process.stdout.write = ((chunk: string | Uint8Array) => { - output += chunk.toString(); - return true; - }) as typeof process.stdout.write; - try { - runE2eWorkflowPlanCli(["--jobs", "hermes-e2e"]); - } finally { - process.stdout.write = write; - } + const output = captureWorkflowPlanCli(["--jobs", "hermes-e2e"]); expect(output.endsWith("\n")).toBe(true); expect(output.trim().split("\n")).toHaveLength(1); @@ -1229,65 +1232,50 @@ describe("E2E workflow plan", () => { }); it("renders the selected targets and workflow jobs as a readable plan", () => { - const filtered = spawnSync(TSX, [PLANNER_CLI, "--summary", "--jobs", "hermes-e2e"], { - cwd: REPO_ROOT, - encoding: "utf8", - timeout: 30_000, - }); + const filtered = captureWorkflowPlanCli(["--summary", "--jobs", "hermes-e2e"]); - expect(filtered.status, filtered.stderr).toBe(0); - expect(filtered.stdout).toBe(`## E2E Execution Plan + expect(filtered).toBe(`## E2E Execution Plan | Target or job | Agent runtime | Observable outcome | Environment or inference endpoint | Source | Unresolved reason | | --- | --- | --- | --- | --- | --- | | \`hermes-e2e\` | hermes | Install onboarding health inference lifecycle dashboard and security succeed | Ubuntu; mock or NVIDIA hosted inference | retained-workflow | | `); - const complete = spawnSync(TSX, [PLANNER_CLI, "--summary"], { - cwd: REPO_ROOT, - encoding: "utf8", - timeout: 30_000, - }); + const complete = captureWorkflowPlanCli(["--summary"]); - expect(complete.status, complete.stderr).toBe(0); - expect(complete.stdout).toContain( + expect(complete).toContain( "| `cloud-onboard` | openclaw | Public install onboarding hosted inference and security checks succeed | Ubuntu; NVIDIA hosted inference | retained-workflow | |", ); - expect(complete.stdout).toContain( + expect(complete).toContain( "| `ubuntu-repo-cloud-openclaw` | openclaw | Repository install onboarding and hosted inference succeed | Ubuntu Docker host; NVIDIA hosted inference | typed-registry | |", ); - expect(complete.stdout).toContain( + expect(complete).toContain( "| `vllm-docker-storage` | none | vLLM storage gate accepts and rejects the intended host states | Native Linux Docker host; no inference endpoint | shared-e2e | |", ); - expect(complete.stdout).toContain( + expect(complete).toContain( "| `channels-add-remove` | openclaw | Messaging: adds and removes Telegram configuration | Ubuntu; no inference endpoint | catalogue | |", ); - expect(complete.stdout).toContain( + expect(complete).toContain( "| `model-router-provider-routed-inference` | openclaw | Inference: Model Router returns a provider-routed response | Ubuntu; NVIDIA API and Model Router | catalogue | |", ); - expect(complete.stdout).toContain( + expect(complete).toContain( "| `spark-install` | unresolved | Install: leaves NemoClaw and OpenShell usable after standard installation | Ubuntu; NVIDIA hosted inference | catalogue | The test asserts CLI usability but does not assert an agent runtime |", ); - expect(complete.stdout).toContain("### Repeated outcomes with distinct evidence"); - expect(complete.stdout).toContain( + expect(complete).toContain("### Repeated outcomes with distinct evidence"); + expect(complete).toContain( "| Repository install onboarding and hosted inference succeed | `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw` | agent runtime |", ); - expect(complete.stdout).toContain("### Intentional exclusions"); - expect(complete.stdout).toContain( + expect(complete).toContain("### Intentional exclusions"); + expect(complete).toContain( "| `llama-cpp-dgx-spark-qualification` | unresolved | Exact NemoClaw-built llama.cpp image produces protected DGX Spark evidence | NVIDIA DGX Spark GB10; local llama.cpp inference | Explicit dispatch only; excluded from the default release matrix | The protected plan can enable or skip its OpenClaw subqualification |", ); - expect(complete.stdout).toContain("### Unsupported or unresolved typed declarations"); - const inertDeclarationCount = listTargets().filter( - (target) => !liveTargetSupport(target).supported, - ).length; - expect(complete.stdout).toContain( - `The ${inertDeclarationCount} inert typed declarations above`, - ); - expect(complete.stdout).toContain( + expect(complete).toContain("### Unsupported or unresolved typed declarations"); + expect(complete).toMatch(/The \d+ inert typed declarations above/u); + expect(complete).toContain( "| `brev-launchable-cloud-openclaw` | unresolved | unresolved | unresolved | platform 'brev-launchable' is not wired for live fixtures; install 'launchable' is not wired for live fixtures |", ); - expect(complete.stdout).toContain("#8285"); - expect(complete.stdout).toContain("#8286"); + expect(complete).toContain("#8285"); + expect(complete).toContain("#8286"); }); it("keeps CI and readable summary output modes separate", () => { diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index a14b94b91d8..de864165ee7 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -364,6 +364,8 @@ function validateConsumer( expectedNeeds = ["base-image-publication", CLI_ARTIFACT_PRODUCER_JOB]; } else if (jobName === "cloud-onboard") { expectedNeeds = ["base-image-publication", CLI_ARTIFACT_PRODUCER_JOB]; + } else if (jobName === "openshell-credential-generation-window") { + expectedNeeds = ["base-image-publication", CLI_ARTIFACT_PRODUCER_JOB]; } if (!isDeepStrictEqual(job.needs, expectedNeeds)) { errors.push(`${jobName} must depend directly on the CLI artifact producer`); diff --git a/tools/e2e/mcp-workflow-boundary.mts b/tools/e2e/mcp-workflow-boundary.mts index a51f6eeca15..9f303dbd2e0 100644 --- a/tools/e2e/mcp-workflow-boundary.mts +++ b/tools/e2e/mcp-workflow-boundary.mts @@ -841,8 +841,8 @@ function validateCredentialWindowJob( requireEqual( errors, JSON.stringify(jobNeeds(job)), - JSON.stringify(["generate-matrix"]), - `${CREDENTIAL_WINDOW_JOB} must depend only on matrix generation so it can run in parallel`, + JSON.stringify(["base-image-publication", "generate-matrix"]), + `${CREDENTIAL_WINDOW_JOB} must depend on its trusted image and matrix producers without MCP serialization`, ); requireEqual( errors, @@ -873,6 +873,8 @@ function validateCredentialWindowJob( E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint", E2E_ARTIFACT_DIR: `\${{ github.workspace }}/${CREDENTIAL_WINDOW_ARTIFACT_DIR}`, + E2E_MANAGED_IMAGE_REVISION: + "${{ needs.generate-matrix.outputs.managed_image_catalog == '' && needs.base-image-publication.outputs.managed_image_revision || '' }}", NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", NEMOCLAW_OPENSHELL_CHANNEL: "stable", NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1",