From 44702695015667d70395fb7693d20ac08f218fd6 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 16 Aug 2026 04:00:17 -0700 Subject: [PATCH 1/8] fix(e2e): repair main runtime regressions Signed-off-by: Prekshi Vyas --- docs/inference/switch-providers.mdx | 2 + docs/reference/commands.mdx | 1 + .../inference-set-compatible-provider.test.ts | 15 +- .../inference-set-route-containment.ts | 7 + src/lib/actions/inference-set.ts | 13 +- src/lib/onboard.ts | 2 +- .../onboard-orchestration.test.ts | 119 ++++++++ .../managed-workload/onboard-orchestration.ts | 15 +- .../fixtures/compatible-anthropic-switch.ts | 255 ------------------ .../live/hermes-inference-switch-helpers.ts | 2 - test/e2e/live/launch-agent-turn.ts | 29 +- .../live/openclaw-inference-switch.test.ts | 2 - test/e2e/mock-parity.json | 4 + .../compatible-anthropic-switch.test.ts | 252 ----------------- test/e2e/support/launch-agent-turn.test.ts | 36 +-- 15 files changed, 186 insertions(+), 568 deletions(-) create mode 100644 src/lib/onboard/managed-workload/onboard-orchestration.test.ts diff --git a/docs/inference/switch-providers.mdx b/docs/inference/switch-providers.mdx index 9b84f2148c..b809d78d23 100644 --- a/docs/inference/switch-providers.mdx +++ b/docs/inference/switch-providers.mdx @@ -73,6 +73,8 @@ Run the rebuild before relying on the running agent. Use `--no-verify` only when OpenShell cannot verify the target provider at switch time and you have already confirmed its provider and credential. This flag does not bypass shared-gateway compatibility checks. +For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw automatically skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +Endpoint-shape and shared-gateway compatibility checks still apply. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index aa56e91fc1..b08bc18a2b 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -4100,6 +4100,7 @@ If the in-sandbox config sync fails, NemoClaw keeps the gateway and registry ali Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `hermes-provider`, `ollama-local`, and `vllm-local`. Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential. +For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw automatically skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network; endpoint-shape and shared-gateway compatibility checks still apply. When switching to `compatible-endpoint` or `compatible-anthropic-endpoint` from a different provider family, pass `--endpoint-url` with the trusted custom provider URL and, except for the Hermes case below, `--inference-api` with its API family so NemoClaw can persist a complete route identity for rebuild and shared-gateway checks. For a Hermes `compatible-anthropic-endpoint` target, `--inference-api` may be omitted because NemoClaw deterministically selects `openai-completions`; an explicit different API family is rejected. NemoClaw rejects loopback, link-local, private, and internal endpoint addresses, including public hostnames that resolve to a private address. diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index 803e6d6487..ea9788dbe7 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -616,7 +616,6 @@ describe("runInferenceSet compatible providers", () => { { provider: "compatible-anthropic-endpoint", model: "mock-anthropic-model", - noVerify: true, endpointUrl: "http://host.openshell.internal:18767/", credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", inferenceApi: "anthropic-messages", @@ -644,6 +643,20 @@ describe("runInferenceSet compatible providers", () => { nimContainer: null, }); expect(deps.calls.rewriteConfigUrlsWithDnsPinning).not.toHaveBeenCalled(); + expect(captureOpenshell).toHaveBeenCalledWith( + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "compatible-anthropic-endpoint", + "--model", + "mock-anthropic-model", + "--no-verify", + ], + expect.objectContaining({ ignoreError: true }), + ); }); for (const provider of ["compatible-endpoint", "compatible-anthropic-endpoint"]) { diff --git a/src/lib/actions/inference-set-route-containment.ts b/src/lib/actions/inference-set-route-containment.ts index 4625e3559e..fb6d250ad7 100644 --- a/src/lib/actions/inference-set-route-containment.ts +++ b/src/lib/actions/inference-set-route-containment.ts @@ -73,6 +73,13 @@ export interface HttpsPinProviderBinding extends InferenceSetProviderBinding { routeId: string; } +/** OpenShell's host verifier cannot resolve routes exposed only on its sandbox bridge. */ +export function isSandboxBridgeProviderBinding( + binding: InferenceSetProviderBinding | null, +): boolean { + return binding !== null && isAllowedOpenShellSandboxBridgeUrl(new URL(binding.baseUrl)); +} + type EnsureHttpsPinAdapterRoute = (endpointUrl: string) => Promise; export interface PreparedInferenceSetRoute { diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index d0e3482524..ceb1d71434 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -85,6 +85,7 @@ import { type EnsureHttpsPinRuntimeAdapterFn, finalizeInferenceSetRoute, type InferenceSetProviderBinding, + isSandboxBridgeProviderBinding, prepareInferenceSetRoute, type RegistryInferenceMetadata, } from "./inference-set-route-containment"; @@ -936,10 +937,14 @@ async function runInferenceSetWithoutHostLock( // verify. Only a genuinely-unreachable host stack hard-fails here, before the // route is touched. let effectiveNoVerify = options.noVerify === true; - // The adapter origin resolves only from inside the sandbox network. The - // host-side OpenShell verifier cannot resolve host.openshell.internal, so - // adapter registration + local health are the verification boundary. - if (httpsPinProviderBinding) effectiveNoVerify = true; + // Adapter routes and explicit custom routes on NemoClaw's sandbox bridge + // resolve only from inside the sandbox network. The host-side OpenShell + // verifier cannot resolve host.openshell.internal, so its result would be a + // guaranteed false negative. Endpoint validation above remains the trust + // boundary; the live sandbox request verifies actual route reachability. + if (isSandboxBridgeProviderBinding(httpsPinProviderBinding ?? directProviderBinding)) { + effectiveNoVerify = true; + } if (deps.isLocalInferenceProvider(provider)) { const localValidation = deps.validateLocalProvider(provider); if (localValidation.ok) { diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index d266f5d3ab..2f575e915d 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1901,7 +1901,7 @@ async function createSandboxWithBaseImageResolution( const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); const { initialSandboxPolicy, policyTier: resolvedCreatePolicyTier, messagingProviders, gpuRoutePlan, compatibilityPolicyPath, initialGpuRoute, sandboxReadyTimeoutSecs, buildId, dashboardRemoteBindPrepared, legacyBuildContext, launch: { createArgv, effectiveDashboardPort, intendedSandboxStartupCommand, managedBootstrapIdentity, managedStartupRootApplyRequest, prebuild, sandboxEnv, sandboxStartupCommand } } = await managedWorkloadOnboard.prepareOnboardSandboxWorkloadLaunch({ runtime: managedWorkloadRuntime, workload: preparedSandboxWorkload, - legacy: { preparedBuildContext, agent, fromDockerfile, createAgentSandbox: (selectedAgent) => baseImageResolutionFlow.createAgentSandboxWithResolution(baseImageResolutionContext, selectedAgent, agentOnboard.createAgentSandbox), patchInput: { preparedBuildContext, agent, fromDockerfile, model, chatUiUrl, provider, endpointUrl: createIntent?.endpointUrl ?? null, compatibleEndpointReasoning: createIntent?.compatibleEndpointReasoning, preferredInferenceApi, webSearchConfig, toolDisclosure: effectiveToolDisclosure, rebuildPreservedEnv: createIntent?.rebuildPreservedEnv, ...(isManagedDcodeAgent ? { dcodeAutoApprovalMode: dcodeAutoApprovalPlan.mode } : {}), hermesToolGateways, sandboxGpuConfig: effectiveSandboxGpuConfig, ...baseImageResolutionFlow.getBaseImageResolutionPatchOptions(baseImageResolutionContext), gatewayPort: GATEWAY_PORT } }, + legacy: { preparedBuildContext, agent, fromDockerfile, createAgentSandbox: (selectedAgent) => baseImageResolutionFlow.createAgentSandboxWithResolution(baseImageResolutionContext, selectedAgent, agentOnboard.createAgentSandbox), resolvePatchInput: () => ({ preparedBuildContext, agent, fromDockerfile, model, chatUiUrl, provider, endpointUrl: createIntent?.endpointUrl ?? null, compatibleEndpointReasoning: createIntent?.compatibleEndpointReasoning, preferredInferenceApi, webSearchConfig, toolDisclosure: effectiveToolDisclosure, rebuildPreservedEnv: createIntent?.rebuildPreservedEnv, ...(isManagedDcodeAgent ? { dcodeAutoApprovalMode: dcodeAutoApprovalPlan.mode } : {}), hermesToolGateways, sandboxGpuConfig: effectiveSandboxGpuConfig, ...baseImageResolutionFlow.getBaseImageResolutionPatchOptions(baseImageResolutionContext), gatewayPort: GATEWAY_PORT }) }, plan: { intent: resolvedCreateIntent, rebindMessagingTokenDefs: async () => (await sandboxCreateIntentResolver.rebind({ sandboxName, enabledChannels, webSearchConfig, agent, ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}) }, resolvedCreateIntent)).messagingTokenDefs, runProviderPreDeleteCleanup: () => runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact, tolerateMissingSandbox: true }), upsertMessagingProviders, getHermesToolGatewayProviderName: (targetSandbox) => getHermesToolGatewayBroker().getHermesToolGatewayProviderName(targetSandbox), discloseInitialSandboxPolicy }, launchInput: { agent, observabilityEnabled: createIntent?.observabilityEnabled === true, chatUiUrl, sandboxName, env: process.env, extraPlaceholderKeys: resolvedCreateIntent.extraPlaceholderKeys, getDashboardForwardPort, hermesDashboardState, hermesApiPort: hermesApiPortReservationScope.effectivePort, manageDashboard, openshellShellCommand, openshellArgv }, plannedMessagingPlan: plannedMessagingState?.plan ?? null, diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts new file mode 100644 index 0000000000..d9b3d312ae --- /dev/null +++ b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { prepareOnboardSandboxWorkloadLaunch } from "./onboard-orchestration"; + +describe("managed workload onboard orchestration", () => { + it("resolves final-image patch metadata after managed build-context staging", async () => { + const resolutionMetadata = { key: "published-dcode-base" }; + let staged = false; + const resolvePatchInput = vi.fn(() => { + expect(staged).toBe(true); + return { preResolvedBaseImageMetadata: resolutionMetadata } as never; + }); + const resolveSandboxBuildPatch = vi.fn(async (input: Record) => { + expect(input.preResolvedBaseImageMetadata).toBe(resolutionMetadata); + expect(input.stagedDockerfile).toBe("/tmp/nemoclaw-staged-context/Dockerfile"); + return { buildId: "dcode-build", dashboardRemoteBindPrepared: false }; + }); + const materializeSandboxCreatePlan = vi.fn(() => ({ + activeMessagingChannels: [], + compatibilityPolicyPath: null, + createArgs: [ + "--from", + "/tmp/nemoclaw-staged-context/Dockerfile", + "--name", + "dcode", + "--policy", + "/tmp/nemoclaw-policy.yaml", + ], + gpuRoutePlan: "none", + initialSandboxPolicy: { + appliedPresets: [], + policyPath: "/tmp/nemoclaw-policy.yaml", + }, + messagingProviders: [], + policyTier: null, + sandboxGpuLogMessage: null, + })); + + await prepareOnboardSandboxWorkloadLaunch({ + runtime: { + runtimeProvider: null, + ensurePreparedWorkload: vi.fn(), + ensurePreparedProfile: vi.fn(), + }, + workload: { + source: { + kind: "legacy-dockerfile", + dockerfilePath: "agents/langchain-deepagents-code/Dockerfile", + reason: "runtime-unsupported", + }, + release: "v0.0.0", + fallbackDiagnostic: null, + }, + legacy: { + preparedBuildContext: null, + agent: { + name: "langchain-deepagents-code", + displayName: "LangChain Deep Agents Code", + }, + fromDockerfile: null, + createAgentSandbox: () => { + staged = true; + return { + buildCtx: "/tmp/nemoclaw-staged-context", + stagedDockerfile: "/tmp/nemoclaw-staged-context/Dockerfile", + baseImageResolutionMetadata: resolutionMetadata, + }; + }, + resolvePatchInput, + }, + plan: { + intent: {}, + rebindMessagingTokenDefs: async () => [], + runProviderPreDeleteCleanup: vi.fn(), + upsertMessagingProviders: vi.fn(() => []), + getHermesToolGatewayProviderName: vi.fn(() => "unused"), + discloseInitialSandboxPolicy: vi.fn(), + }, + launchInput: { + agent: null, + chatUiUrl: "http://127.0.0.1:18789", + sandboxName: "dcode", + env: { NEMOCLAW_SANDBOX_PREBUILD: "0" }, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: {}, + manageDashboard: false, + openshellShellCommand: () => "openshell sandbox create", + }, + plannedMessagingPlan: null, + gpu: { + provider: "compatible-endpoint", + config: { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }, + dockerDriverGateway: false, + gatewayPort: 8080, + }, + dependencies: { + materializeSandboxCreatePlan, + prepareSandboxBuildPatchConfig: vi.fn(() => ({ + messagingChannelConfig: null, + })), + resolveSandboxBuildPatch, + }, + } as unknown as Parameters[0]); + + expect(resolvePatchInput).toHaveBeenCalledOnce(); + expect(resolveSandboxBuildPatch).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index e28c25ef83..8ebcdd25df 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -245,7 +245,10 @@ export interface PrepareOnboardSandboxWorkloadLaunchInput { readonly createAgentSandbox: ( agent: AgentDefinition, ) => ReturnType; - readonly patchInput: Omit; + readonly resolvePatchInput: () => Omit< + ResolveBuildPatchInput, + "selectedGpuRoute" | "stagedDockerfile" + >; }; readonly plan: { readonly intent: SandboxCreateIntent; @@ -269,6 +272,7 @@ export interface PrepareOnboardSandboxWorkloadLaunchInput { readonly dependencies: { readonly materializeSandboxCreatePlan: typeof import("../sandbox-create-plan-materialization").materializeSandboxCreatePlan; readonly prepareSandboxBuildPatchConfig: typeof import("../sandbox-build-patch-config").prepareSandboxBuildPatchConfig; + readonly resolveSandboxBuildPatch?: typeof import("../prepared-dcode-rebuild").resolveSandboxBuildPatch; }; readonly log?: (message: string) => void; readonly onExit?: (cleanup: () => void) => void; @@ -377,8 +381,13 @@ export async function prepareOnboardSandboxWorkloadLaunch( } else { const buildContext = requireLegacyBuildContext(legacyBuildContext); input.dependencies.prepareSandboxBuildPatchConfig({ configuredMessagingChannels }); - const patch = await resolveSandboxBuildPatch({ - ...input.legacy.patchInput, + const patch = await ( + input.dependencies.resolveSandboxBuildPatch ?? resolveSandboxBuildPatch + )({ + // Build-context staging resolves managed-agent base-image provenance. + // Read the patch input only after that boundary so the final image gets + // the exact metadata produced by the same staging operation. + ...input.legacy.resolvePatchInput(), selectedGpuRoute: initialGpuRoute, stagedDockerfile: buildContext.stagedDockerfile, }); diff --git a/test/e2e/fixtures/compatible-anthropic-switch.ts b/test/e2e/fixtures/compatible-anthropic-switch.ts index d80c96d29d..59d019ad81 100644 --- a/test/e2e/fixtures/compatible-anthropic-switch.ts +++ b/test/e2e/fixtures/compatible-anthropic-switch.ts @@ -1,13 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { randomBytes } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { parseDockerDriverGatewayRuntimeMarker } from "../../../src/lib/onboard/docker-driver-gateway-runtime-marker.ts"; -import { resolveDockerDriverGatewayStateDir } from "../../../src/lib/onboard/host-gateway-process.ts"; import type { HostCliClient } from "./clients/host.ts"; import { resultText } from "./clients/index.ts"; @@ -15,71 +8,6 @@ export const COMPATIBLE_ANTHROPIC_PROVIDER = "compatible-anthropic-endpoint"; export const COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV = "COMPATIBLE_ANTHROPIC_API_KEY"; const DEFAULT_COMPATIBLE_ANTHROPIC_CREDENTIAL = "test-compatible-anthropic-key"; const OPENSHELL_HOST_ALIAS = "host.openshell.internal"; -const GATEWAY_SERVICE_NAMES = ["nemoclaw-openshell-gateway", "openshell-gateway"] as const; -const GATEWAY_STATE_FILE_LIMIT = 64 * 1024; - -export const GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT = [ - "set -euo pipefail", - 'operation="$1"', - 'resolver_source="$2"', - 'owner_token="$3"', - 'hosts_path="$4"', - `alias_name="${OPENSHELL_HOST_ALIAS}"`, - 'owned_line="127.0.0.1 ${alias_name} # nemoclaw-gateway-host-verifier:${owner_token}"', - "", - 'case "$operation" in', - " add | remove) ;;", - ' *) echo "unsupported gateway resolver operation: $operation" >&2; exit 2 ;;', - "esac", - '[[ "$owner_token" =~ ^[a-f0-9]{32}$ ]] || { echo "invalid gateway resolver owner token" >&2; exit 2; }', - '[[ -f "$hosts_path" ]] || { echo "gateway resolver path is not a regular file" >&2; exit 2; }', - "", - 'if [[ "$operation" == "add" ]]; then', - ' [[ -f "$resolver_source" && ! -L "$resolver_source" ]] || { echo "gateway resolver source is not a regular file" >&2; exit 2; }', - ' grep -Fqx -- "$owned_line" "$resolver_source" || { echo "gateway resolver source lacks its ownership marker" >&2; exit 2; }', - " mount --make-rprivate /", - ' mount --bind "$resolver_source" "$hosts_path"', - ' grep -Fqx -- "$owned_line" "$hosts_path" || { echo "gateway resolver mount was not installed" >&2; exit 4; }', - " exit 0", - "fi", - "", - '# A restarted gateway has already released the owned mount namespace.', - 'grep -Fqx -- "$owned_line" "$hosts_path" || exit 0', - 'umount "$hosts_path"', - 'if grep -Fqx -- "$owned_line" "$hosts_path"; then', - ' echo "gateway resolver mount was not removed" >&2', - " exit 4", - "fi", -].join("\n"); - -const GATEWAY_HOST_VERIFICATION_NAMESPACE_SCRIPT = [ - "set -euo pipefail", - 'operation="$1"', - 'target_pid="$2"', - 'resolver_source="$3"', - 'owner_token="$4"', - 'mount_script="$5"', - "", - '[[ "$target_pid" =~ ^[1-9][0-9]*$ ]] || { echo "invalid OpenShell gateway PID" >&2; exit 2; }', - "gateway_is_alive() {", - ' local executable=""', - ' [[ -r "/proc/${target_pid}/stat" ]] || return 1', - ' executable="$(readlink -f "/proc/${target_pid}/exe" 2>/dev/null || true)"', - ' [[ "${executable##*/}" == "openshell-gateway" ]]', - "}", - "", - 'if ! gateway_is_alive; then', - ' [[ "$operation" == "remove" ]] && exit 0', - ' echo "active OpenShell gateway process is unavailable" >&2', - " exit 3", - "fi", - 'command -v nsenter >/dev/null 2>&1 || { echo "nsenter is required for scoped gateway resolution" >&2; exit 2; }', - 'current_namespace="$(readlink /proc/self/ns/mnt)"', - 'target_namespace="$(readlink "/proc/${target_pid}/ns/mnt")"', - '[[ "$current_namespace" != "$target_namespace" ]] || { echo "OpenShell gateway does not have a private mount namespace" >&2; exit 3; }', - "", - 'exec nsenter --target "$target_pid" --mount -- bash -ceu "$mount_script" gateway-resolver-mount "$operation" "$resolver_source" "$owner_token" /etc/hosts', -].join("\n"); export interface CompatibleAnthropicSwitchBinding { endpointUrl: string; @@ -90,189 +18,6 @@ export function compatibleAnthropicMockEndpointUrl(port: number): string { return `http://${OPENSHELL_HOST_ALIAS}:${port}`; } -function pathExists(filePath: string): boolean { - try { - fs.lstatSync(filePath); - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; - throw error; - } -} - -function readOwnedGatewayStateFile(filePath: string, currentUid: number): string | null { - if (typeof fs.constants.O_NOFOLLOW !== "number") return null; - let descriptor: number | undefined; - try { - descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); - const stat = fs.fstatSync(descriptor); - if ( - !stat.isFile() || - stat.nlink !== 1 || - stat.uid !== currentUid || - (stat.mode & 0o022) !== 0 || - stat.size > GATEWAY_STATE_FILE_LIMIT - ) { - return null; - } - return fs.readFileSync(descriptor, "utf8"); - } catch { - return null; - } finally { - if (descriptor !== undefined) fs.closeSync(descriptor); - } -} - -function managedOpenShellGatewayPid(homeDir: string): number | null { - const stateDirectory = resolveDockerDriverGatewayStateDir(process.env, homeDir); - const pidPath = path.join(stateDirectory, "openshell-gateway.pid"); - const markerPath = path.join(stateDirectory, "runtime.json"); - const pidPathExists = pathExists(pidPath); - const markerPathExists = pathExists(markerPath); - if (!pidPathExists && !markerPathExists) return null; - if (!pidPathExists || !markerPathExists) { - throw new Error("Docker-driver gateway state is incomplete"); - } - - const currentUid = process.getuid?.(); - if (currentUid === undefined) { - throw new Error("Docker-driver gateway state ownership is unavailable"); - } - const pidText = readOwnedGatewayStateFile(pidPath, currentUid); - const markerText = readOwnedGatewayStateFile(markerPath, currentUid); - if (!pidText || !markerText) { - throw new Error("Docker-driver gateway state is not an owned regular file"); - } - if (!/^[1-9][0-9]*\n?$/u.test(pidText)) { - throw new Error("Docker-driver gateway PID file is invalid"); - } - const pid = Number(pidText.trim()); - const marker = parseDockerDriverGatewayRuntimeMarker(markerText); - if ( - !Number.isSafeInteger(pid) || - !marker || - marker.pid !== pid || - marker.platform !== process.platform || - marker.arch !== process.arch - ) { - throw new Error("Docker-driver gateway state does not identify the current process"); - } - if (marker.endpoint !== "https://127.0.0.1:8080") { - throw new Error("Docker-driver gateway state does not identify the default gateway"); - } - - let processStat: fs.Stats; - let executable: string; - try { - processStat = fs.statSync(`/proc/${pid}`); - executable = fs.realpathSync(`/proc/${pid}/exe`); - } catch { - throw new Error("Docker-driver gateway process is unavailable"); - } - if (processStat.uid !== currentUid || path.basename(executable) !== "openshell-gateway") { - throw new Error("Docker-driver gateway process identity does not match its state"); - } - if (marker.gatewayBin) { - let recordedExecutable: string; - try { - recordedExecutable = fs.realpathSync(marker.gatewayBin); - } catch { - throw new Error("Docker-driver gateway executable is unavailable"); - } - if (executable !== recordedExecutable) { - throw new Error("Docker-driver gateway executable does not match its state"); - } - } - return pid; -} - -async function activeOpenShellGatewayPid(host: HostCliClient, homeDir: string): Promise { - const managedPid = managedOpenShellGatewayPid(homeDir); - if (managedPid !== null) return managedPid; - for (const serviceName of GATEWAY_SERVICE_NAMES) { - const result = await host.command( - "systemctl", - [ - "--user", - "show", - serviceName, - "--property=ActiveState", - "--property=MainPID", - ], - { artifactName: `inspect-${serviceName}`, timeoutMs: 30_000 }, - ); - if (result.exitCode !== 0) continue; - const properties = new Map( - result.stdout - .split(/\r?\n/u) - .map((line) => line.split("=", 2)) - .filter((entry): entry is [string, string] => entry.length === 2), - ); - const pid = Number(properties.get("MainPID")); - if (properties.get("ActiveState") === "active" && Number.isSafeInteger(pid) && pid > 0) { - return pid; - } - } - throw new Error("could not find an active OpenShell gateway user service"); -} - -export async function installGatewayHostVerificationAlias( - host: HostCliClient, - cleanup: { add(name: string, run: () => Promise | void): void }, - homeDir: string = os.homedir(), -): Promise { - const gatewayPid = await activeOpenShellGatewayPid(host, homeDir); - const ownerToken = randomBytes(16).toString("hex"); - const fixtureDirectory = fs.mkdtempSync( - path.join(homeDir, ".nemoclaw-gateway-resolver-"), - ); - const resolverSource = path.join(fixtureDirectory, "hosts"); - const ownedLine = `127.0.0.1 ${OPENSHELL_HOST_ALIAS} # nemoclaw-gateway-host-verifier:${ownerToken}`; - fs.chmodSync(fixtureDirectory, 0o700); - fs.writeFileSync(resolverSource, `${ownedLine}\n${fs.readFileSync("/etc/hosts", "utf8")}`, { - mode: 0o600, - }); - - const updateMount = async (operation: "add" | "remove"): Promise => { - const result = await host.command( - "sudo", - [ - "bash", - "-ceu", - GATEWAY_HOST_VERIFICATION_NAMESPACE_SCRIPT, - `gateway-resolver-${operation}`, - operation, - String(gatewayPid), - resolverSource, - ownerToken, - GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT, - ], - { artifactName: `${operation}-gateway-host-verifier-alias`, timeoutMs: 60_000 }, - ); - if (result.exitCode !== 0) { - throw new Error( - `could not ${operation === "add" ? "install" : "remove"} the gateway host verifier alias: ${resultText(result)}`, - ); - } - }; - - let restored = false; - const restore = async (): Promise => { - if (restored) return; - await updateMount("remove"); - restored = true; - fs.rmSync(fixtureDirectory, { force: true, recursive: true }); - }; - cleanup.add("remove the OpenShell gateway resolver mount", restore); - - try { - await updateMount("add"); - } catch (error) { - await restore(); - throw error; - } -} - export function compatibleAnthropicSwitchBinding( endpointUrl: string, runtimeEnv: NodeJS.ProcessEnv = process.env, diff --git a/test/e2e/live/hermes-inference-switch-helpers.ts b/test/e2e/live/hermes-inference-switch-helpers.ts index d7aff6030a..f485101de4 100644 --- a/test/e2e/live/hermes-inference-switch-helpers.ts +++ b/test/e2e/live/hermes-inference-switch-helpers.ts @@ -21,7 +21,6 @@ import { compatibleAnthropicMockEndpointUrl, compatibleAnthropicSwitchBinding, compatibleAnthropicSwitchEnv, - installGatewayHostVerificationAlias, requireCompatibleAnthropicProviderAbsent, } from "../fixtures/compatible-anthropic-switch.ts"; import { expect } from "../fixtures/e2e-test.ts"; @@ -611,7 +610,6 @@ export async function prepareCompatibleAnthropicSwitchBinding( return null; const mock = mockAnthropicSwitchEnabled() ? await startMockAnthropicProvider() : undefined; mock && cleanup.add("close compatible Anthropic switch mock", () => mock.close()); - if (mock) await installGatewayHostVerificationAlias(host, cleanup); const binding = compatibleAnthropicSwitchBinding( process.env.NEMOCLAW_SWITCH_ENDPOINT_URL ?? mock?.endpointUrl ?? "", ); diff --git a/test/e2e/live/launch-agent-turn.ts b/test/e2e/live/launch-agent-turn.ts index 6a9cd73afd..542bfeb0e9 100644 --- a/test/e2e/live/launch-agent-turn.ts +++ b/test/e2e/live/launch-agent-turn.ts @@ -13,7 +13,6 @@ import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; // baseline. Session content never moves to the host. export const OPENCLAW_SESSION_EVIDENCE_SCRIPT = String.raw` const crypto = require("node:crypto"); -const childProcess = require("node:child_process"); const fs = require("node:fs"); const path = require("node:path"); @@ -71,7 +70,11 @@ function openClawTuiProcessIds() { return pids; } -function qualifyTuiInputMode() { +// The terminal line discipline safely queues a complete submitted line even +// while a canonical-mode TUI is still installing its reader. Raw mode is a UI +// implementation detail, so the stable readiness boundary is the one matching +// OpenClaw TUI process owning the launch PTY on standard input. +function qualifyTuiInputPty() { const pids = openClawTuiProcessIds(); if (pids.length === 0) finish(1); if (pids.length > 1) finish(2, "multiple_tui_processes"); @@ -83,16 +86,6 @@ function qualifyTuiInputMode() { finish(2, "tui_stdin_unavailable"); } if (!/^\/dev\/pts\/\d+$/.test(ttyPath)) finish(2, "tui_stdin_not_pty"); - let state; - try { - state = childProcess.execFileSync("stty", ["-F", ttyPath, "-a"], { - encoding: "utf8", - stdio: ["ignore", "pipe", "ignore"], - }); - } catch { - finish(2, "tui_termios_unavailable"); - } - if (!/(^|[\s;])-icanon([\s;]|$)/.test(state)) finish(1); finish(0); } @@ -228,7 +221,7 @@ function qualifyTurns() { try { if (mode === "baseline") recordBaseline(); - if (mode === "input-mode") qualifyTuiInputMode(); + if (mode === "input-pty") qualifyTuiInputPty(); if (mode === "qualify") qualifyTurns(); } catch { finish(2, "verifier_failed"); @@ -346,23 +339,23 @@ wait_for_turn_count() { fail_launch_session "launch did not record the required structured session turns" } -wait_for_pty_input_mode() { +wait_for_tui_input_pty() { local evidence_status while (( SECONDS < session_deadline )); do - if session_evidence input-mode >/dev/null 2>"$evidence_error"; then + if session_evidence input-pty >/dev/null 2>"$evidence_error"; then return 0 else evidence_status=$? fi if [[ "$evidence_status" != 1 ]]; then - fail_launch_session "OpenClaw TUI input-mode evidence was invalid or unavailable (status $evidence_status)" + fail_launch_session "OpenClaw TUI input PTY evidence was invalid or unavailable (status $evidence_status)" fi if ! kill -0 "$session_pid" 2>/dev/null; then break fi sleep 0.1 done - fail_launch_session "launch PTY did not enter input mode before the session deadline" + fail_launch_session "OpenClaw TUI did not attach standard input to the launch PTY before the session deadline" } if ! session_evidence baseline >/dev/null 2>"$evidence_error"; then @@ -402,7 +395,7 @@ if [[ "$capture_ready" != 1 ]]; then fail_launch_session "launch did not create a PTY diagnostic capture" fi -wait_for_pty_input_mode +wait_for_tui_input_pty if ! printf '%s\r' "$NEMOCLAW_LAUNCH_FIRST_INPUT" >&3; then fail_launch_session "launch exited before the first PTY input was submitted" fi diff --git a/test/e2e/live/openclaw-inference-switch.test.ts b/test/e2e/live/openclaw-inference-switch.test.ts index 298d3c9356..2e81df9fc1 100644 --- a/test/e2e/live/openclaw-inference-switch.test.ts +++ b/test/e2e/live/openclaw-inference-switch.test.ts @@ -30,7 +30,6 @@ import { compatibleAnthropicMockEndpointUrl, compatibleAnthropicSwitchBinding, compatibleAnthropicSwitchEnv, - installGatewayHostVerificationAlias, requireCompatibleAnthropicProviderAbsent, } from "../fixtures/compatible-anthropic-switch.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -1093,7 +1092,6 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha if (SWITCH_PROVIDER === "compatible-anthropic-endpoint" && SWITCH_MOCK_ANTHROPIC === "1") { mockProvider = await startMockAnthropicProvider(); - await installGatewayHostVerificationAlias(host, cleanup, home); await artifacts.writeJson("mock-anthropic-provider.json", { endpointUrl: mockProvider.endpointUrl, }); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index bbfec6b1fa..25d7feac17 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -364,6 +364,8 @@ { "live": "test/e2e/live/hermes-inference-switch.test.ts", "fast": [ + "src/lib/actions/inference-set-compatible-provider.test.ts", + "test/e2e/support/compatible-anthropic-switch.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -448,6 +450,8 @@ { "live": "test/e2e/live/openclaw-inference-switch.test.ts", "fast": [ + "src/lib/actions/inference-set-compatible-provider.test.ts", + "test/e2e/support/compatible-anthropic-switch.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] diff --git a/test/e2e/support/compatible-anthropic-switch.test.ts b/test/e2e/support/compatible-anthropic-switch.test.ts index 0c56155985..2afeb72495 100644 --- a/test/e2e/support/compatible-anthropic-switch.test.ts +++ b/test/e2e/support/compatible-anthropic-switch.test.ts @@ -1,73 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - import { afterEach, describe, expect, it, vi } from "vitest"; import { normalizeCustomEndpointUrl } from "../../../src/lib/actions/inference-set.ts"; -import { - writeDockerDriverGatewayPidFile, - writeDockerDriverGatewayRuntimeMarkerForStateDir, -} from "../../../src/lib/onboard/docker-driver-gateway-runtime-marker.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV, COMPATIBLE_ANTHROPIC_PROVIDER, - GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT, compatibleAnthropicMockEndpointUrl, compatibleAnthropicSwitchBinding, compatibleAnthropicSwitchEnv, - installGatewayHostVerificationAlias, requireCompatibleAnthropicProviderAbsent, } from "../fixtures/compatible-anthropic-switch.ts"; -const INVALID_MANAGED_GATEWAY_STATE_CASES = [ - { - label: "invalid", - pid: process.pid, - writePid: (stateDirectory: string, _pid: number) => - fs.writeFileSync(path.join(stateDirectory, "openshell-gateway.pid"), "not-a-pid\n", { - mode: 0o600, - }), - }, - { - label: "symlinked", - pid: process.pid, - writePid: (stateDirectory: string, pid: number) => { - const target = path.join(stateDirectory, "pid-target"); - fs.writeFileSync(target, `${pid}\n`, { mode: 0o600 }); - fs.symlinkSync(target, path.join(stateDirectory, "openshell-gateway.pid")); - }, - }, - { - label: "stale", - pid: 2_147_483_647, - writePid: (stateDirectory: string, pid: number) => - writeDockerDriverGatewayPidFile( - path.join(stateDirectory, "openshell-gateway.pid"), - pid, - ), - }, -] as const; - -function mockGatewayProcess(pid: number, gatewayBin: string): void { - const statSync = fs.statSync; - vi.spyOn(fs, "statSync").mockImplementation(((target) => - String(target) === `/proc/${pid}` - ? ({ uid: process.getuid?.() ?? 0 } as fs.Stats) - : statSync(target)) as typeof fs.statSync); - const realpathSync = fs.realpathSync; - const gatewayExecutablePaths = new Set([`/proc/${pid}/exe`, gatewayBin]); - vi.spyOn(fs, "realpathSync").mockImplementation(((target) => - gatewayExecutablePaths.has(String(target)) - ? gatewayBin - : realpathSync(target)) as typeof fs.realpathSync); -} - describe("compatible Anthropic inference switch setup", () => { afterEach(() => { vi.unstubAllEnvs(); @@ -113,127 +59,6 @@ describe("compatible Anthropic inference switch setup", () => { expect(rewrite).not.toHaveBeenCalled(); }); - it("uses managed Docker-driver gateway state from the target home before the user service (#9166)", async () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-target-home-gateway-test-")); - const stateDirectory = path.join( - home, - ".local", - "state", - "nemoclaw", - "openshell-docker-gateway", - ); - const pid = process.pid; - const gatewayBin = "/usr/bin/openshell-gateway"; - fs.mkdirSync(stateDirectory, { recursive: true }); - vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", ""); - writeDockerDriverGatewayPidFile(path.join(stateDirectory, "openshell-gateway.pid"), pid); - writeDockerDriverGatewayRuntimeMarkerForStateDir(stateDirectory, { - desiredEnv: {}, - endpoint: "https://127.0.0.1:8080", - gatewayBin, - pid, - }); - mockGatewayProcess(pid, gatewayBin); - const command = vi.fn().mockResolvedValue({ exitCode: 0, stderr: "", stdout: "" }); - const add = vi.fn(); - - try { - await installGatewayHostVerificationAlias( - { command } as unknown as HostCliClient, - { add }, - home, - ); - const cleanupMount = add.mock.calls[0]?.[1] as () => Promise; - await cleanupMount(); - - expect(command).toHaveBeenCalledTimes(2); - for (const call of command.mock.calls) { - expect(call[0]).toBe("sudo"); - expect(call[1]).toEqual( - expect.arrayContaining([String(pid), GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT]), - ); - } - } finally { - fs.rmSync(home, { force: true, recursive: true }); - } - }); - - it("uses the active user service when managed gateway state is absent (#9166)", async () => { - const stateDirectory = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-missing-gateway-state-test-"), - ); - vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", stateDirectory); - const command = vi - .fn() - .mockResolvedValueOnce({ - exitCode: 0, - stderr: "", - stdout: "ActiveState=active\nMainPID=4242\n", - }) - .mockResolvedValue({ exitCode: 0, stderr: "", stdout: "" }); - const add = vi.fn(); - - try { - await installGatewayHostVerificationAlias( - { command } as unknown as HostCliClient, - { add }, - stateDirectory, - ); - const cleanupMount = add.mock.calls[0]?.[1] as () => Promise; - await cleanupMount(); - - expect(command.mock.calls[0]?.slice(0, 2)).toEqual([ - "systemctl", - [ - "--user", - "show", - "nemoclaw-openshell-gateway", - "--property=ActiveState", - "--property=MainPID", - ], - ]); - for (const call of command.mock.calls.slice(1)) { - expect(call[0]).toBe("sudo"); - expect(call[1]).toEqual( - expect.arrayContaining(["4242", GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT]), - ); - } - } finally { - fs.rmSync(stateDirectory, { force: true, recursive: true }); - } - }); - - it.each(INVALID_MANAGED_GATEWAY_STATE_CASES)( - "rejects $label managed gateway PID state (#9166)", - async ({ pid, writePid }) => { - const stateDirectory = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-invalid-gateway-state-test-"), - ); - vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", stateDirectory); - writeDockerDriverGatewayRuntimeMarkerForStateDir(stateDirectory, { - desiredEnv: {}, - endpoint: "https://127.0.0.1:8080", - gatewayBin: "/usr/bin/openshell-gateway", - pid, - }); - writePid(stateDirectory, pid); - const command = vi.fn(); - - try { - await expect( - installGatewayHostVerificationAlias({ command } as unknown as HostCliClient, { - add: vi.fn(), - }), - ).rejects.toThrow( - /Docker-driver gateway (PID file is invalid|process is unavailable|state is not an owned regular file)/u, - ); - expect(command).not.toHaveBeenCalled(); - } finally { - fs.rmSync(stateDirectory, { force: true, recursive: true }); - } - }, - ); - it("requires the direct provider to be absent before inference set owns its creation", async () => { const command = vi.fn().mockResolvedValue({ exitCode: 1, @@ -283,80 +108,3 @@ describe("compatible Anthropic inference switch setup", () => { ); }); }); - -const linuxIt = process.platform === "linux" ? it : it.skip; - -describe("gateway resolver mount", () => { - linuxIt("preserves a resolver write that overlaps mount installation (#9166)", () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-resolver-test-")); - const hostsPath = path.join(directory, "hosts"); - const underlayPath = path.join(directory, "hosts.underlay"); - const resolverSource = path.join(directory, "resolver-source"); - const fakeBin = path.join(directory, "bin"); - const token = "a".repeat(32); - const ownedLine = - `127.0.0.1 host.openshell.internal # nemoclaw-gateway-host-verifier:${token}`; - try { - fs.mkdirSync(fakeBin); - fs.writeFileSync(hostsPath, "127.0.0.1 localhost\n", { mode: 0o644 }); - fs.writeFileSync(resolverSource, `${ownedLine}\n127.0.0.1 localhost\n`, { mode: 0o600 }); - fs.writeFileSync( - path.join(fakeBin, "mount"), - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - '[[ "$1" == "--make-rprivate" ]] && exit 0', - '[[ "$1" == "--bind" ]]', - "printf '192.0.2.10 concurrent.example.test\\n' >> \"$3\"", - 'mv -- "$3" "$NEMOCLAW_TEST_RESOLVER_UNDERLAY"', - 'ln -s -- "$2" "$3"', - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(fakeBin, "umount"), - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - 'rm -- "$1"', - 'mv -- "$NEMOCLAW_TEST_RESOLVER_UNDERLAY" "$1"', - ].join("\n"), - { mode: 0o755 }, - ); - const run = (operation: "add" | "remove") => - spawnSync( - "bash", - [ - "-ceu", - GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT, - "gateway-resolver-mount-test", - operation, - resolverSource, - token, - hostsPath, - ], - { - encoding: "utf8", - env: { - ...process.env, - NEMOCLAW_TEST_RESOLVER_UNDERLAY: underlayPath, - PATH: `${fakeBin}:${process.env.PATH ?? ""}`, - }, - }, - ); - - const added = run("add"); - expect(added.status, added.stderr).toBe(0); - expect(fs.readFileSync(hostsPath, "utf8")).toContain(ownedLine); - expect(fs.readFileSync(underlayPath, "utf8")).toContain("concurrent.example.test"); - - const removed = run("remove"); - expect(removed.status, removed.stderr).toBe(0); - expect(fs.readFileSync(hostsPath, "utf8")).toBe( - "127.0.0.1 localhost\n192.0.2.10 concurrent.example.test\n", - ); - } finally { - fs.rmSync(directory, { force: true, recursive: true }); - } - }); -}); diff --git a/test/e2e/support/launch-agent-turn.test.ts b/test/e2e/support/launch-agent-turn.test.ts index 8c319e75b7..3fbcb03a62 100644 --- a/test/e2e/support/launch-agent-turn.test.ts +++ b/test/e2e/support/launch-agent-turn.test.ts @@ -28,9 +28,8 @@ const PROCESS_EXIT_WAIT = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_ type SessionRecords = Record; type FixtureMode = | "cleanup-failure" - | "delayed-input-attachment" + | "delayed-input-reader" | "delayed-recording" - | "input-mode-timeout" | "invalid-order" | "late-extra" | "multiple-tui-processes" @@ -256,13 +255,8 @@ if (process.argv[2] === "tui") (async () => { } process.exit(fs.existsSync(process.env.NEMOCLAW_FIXTURE_TUI_STDIN_RETRY) ? 0 : 68); } - if (mode === "delayed-input-attachment" || mode === "input-mode-timeout") { - let inputBeforeAttachment = false; - const recordEarlyInput = () => { inputBeforeAttachment = true; }; - process.stdin.on("data", recordEarlyInput); - await new Promise((resolve) => setTimeout(resolve, mode === "input-mode-timeout" ? 10_000 : 1_500)); - process.stdin.off("data", recordEarlyInput); - if (inputBeforeAttachment) process.exit(67); + if (mode === "delayed-input-reader") { + await new Promise((resolve) => setTimeout(resolve, 1_500)); } const rl = readline.createInterface({ input: process.stdin, output: process.stdout, terminal: true }); const ask = () => new Promise((resolve) => rl.question("", resolve)); @@ -314,7 +308,7 @@ fi while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done [[ "$#" -gt 0 ]] shift -if [[ "$NEMOCLAW_FIXTURE_MODE" == "transient-tui-stdin" && "$4" == "input-mode" ]]; then +if [[ "$NEMOCLAW_FIXTURE_MODE" == "transient-tui-stdin" && "$4" == "input-pty" ]]; then set +e "$@" status=$? @@ -487,10 +481,10 @@ it.runIf(process.platform === "linux")( ); it.runIf(process.platform === "linux")( - "waits for the OpenClaw TUI input mode before submitting PTY input (#9160)", + "queues one canonical PTY line until the OpenClaw TUI installs its input reader (#9160)", () => { const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( - "delayed-input-attachment", + "delayed-input-reader", "absent", ); @@ -563,24 +557,6 @@ it.runIf(process.platform === "linux")( }, ); -it.runIf(process.platform === "linux")( - "reports a missing OpenClaw input mode before the PTY child timeout (#9160)", - () => { - const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( - "input-mode-timeout", - "absent", - ); - - expect(ttyObserved).toBe(true); - expect(baselineRemoved).toBe(true); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "launch PTY did not enter input mode before the session deadline", - ); - }, -); - it.runIf(process.platform === "linux")( "reports missing structured turns before the PTY child timeout (#9160)", () => { From 3edb7353f4b8581665e2a01f68b54132b15c3875 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 16 Aug 2026 04:42:19 -0700 Subject: [PATCH 2/8] fix(e2e): close route and PTY review gaps Signed-off-by: Prekshi Vyas --- docs/inference/switch-providers.mdx | 4 +- docs/reference/commands.mdx | 5 +- .../inference-set-compatible-provider.test.ts | 86 +++++++++++++ src/lib/actions/inference-set-provider.ts | 21 ++++ src/lib/actions/inference-set.test-support.ts | 4 + src/lib/actions/inference-set.ts | 94 +++++++++++--- test/e2e/live/launch-agent-turn.ts | 101 +++++++++++++-- test/e2e/support/launch-agent-turn.test.ts | 119 ++++++++++++++++-- 8 files changed, 393 insertions(+), 41 deletions(-) diff --git a/docs/inference/switch-providers.mdx b/docs/inference/switch-providers.mdx index b809d78d23..001722e700 100644 --- a/docs/inference/switch-providers.mdx +++ b/docs/inference/switch-providers.mdx @@ -73,7 +73,9 @@ Run the rebuild before relying on the running agent. Use `--no-verify` only when OpenShell cannot verify the target provider at switch time and you have already confirmed its provider and credential. This flag does not bypass shared-gateway compatibility checks. -For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw automatically skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +It then sends one minimal request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens. +If that request fails, NemoClaw restores the previous OpenShell selection and removes a provider that this switch created. Endpoint-shape and shared-gateway compatibility checks still apply. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index b08bc18a2b..9b075a9759 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -4100,7 +4100,10 @@ If the in-sandbox config sync fails, NemoClaw keeps the gateway and registry ali Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `hermes-provider`, `ollama-local`, and `vllm-local`. Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential. -For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw automatically skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network; endpoint-shape and shared-gateway compatibility checks still apply. +For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +Before it persists the route in the NemoClaw registry or agent config, the command sends one request from the target sandbox with a 16-token output limit. +If that request fails, the command restores the previous OpenShell selection and removes a provider that this switch created. +Endpoint-shape and shared-gateway compatibility checks still apply. When switching to `compatible-endpoint` or `compatible-anthropic-endpoint` from a different provider family, pass `--endpoint-url` with the trusted custom provider URL and, except for the Hermes case below, `--inference-api` with its API family so NemoClaw can persist a complete route identity for rebuild and shared-gateway checks. For a Hermes `compatible-anthropic-endpoint` target, `--inference-api` may be omitted because NemoClaw deterministically selects `openai-completions`; an explicit different API family is rejected. NemoClaw rejects loopback, link-local, private, and internal endpoint addresses, including public hostnames that resolve to a private address. diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index ea9788dbe7..cc998c94dd 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -657,6 +657,92 @@ describe("runInferenceSet compatible providers", () => { ], expect.objectContaining({ ignoreError: true }), ); + expect(deps.calls.probeSandboxRoute).toHaveBeenCalledWith({ + sandboxName: "alpha", + provider: "compatible-anthropic-endpoint", + model: "mock-anthropic-model", + preferredInferenceApi: "anthropic-messages", + }); + expect(deps.calls.probeSandboxRoute.mock.invocationCallOrder[0]).toBeLessThan( + deps.calls.updateSandbox.mock.invocationCallOrder[0], + ); + }); + + it("restores the prior route when sandbox-only provider verification fails", async () => { + const captureOpenshell = createCompatibleProviderCapture({ + name: "compatible-anthropic-endpoint", + type: "anthropic", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + configKey: "ANTHROPIC_BASE_URL", + initiallyPresent: false, + }); + const deps = createDeps({ + config: { agents: { defaults: { model: { primary: "inference/old-model" } } } }, + entry: { + name: "alpha", + agent: "openclaw", + provider: "nvidia-prod", + model: "old-model", + }, + session: baseSession({ provider: "nvidia-prod", model: "old-model" }), + captureOpenshell, + probeSandboxRoute: () => ({ + ok: false, + detail: "sandbox inference invocation probe exited with status 7", + httpStatus: null, + }), + }); + + await expect( + runInferenceSet( + { + provider: "compatible-anthropic-endpoint", + model: "mock-anthropic-model", + endpointUrl: "http://host.openshell.internal:18767/", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + inferenceApi: "anthropic-messages", + }, + deps, + ), + ).rejects.toThrow( + /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s, + ); + + expect( + captureOpenshell.mock.calls + .filter(([args]) => args[0] === "inference" && args[1] === "set") + .map(([args]) => args), + ).toEqual([ + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "compatible-anthropic-endpoint", + "--model", + "mock-anthropic-model", + "--no-verify", + ], + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "nvidia-prod", + "--model", + "old-model", + "--no-verify", + ], + ]); + expect( + captureOpenshell.mock.calls.some( + ([args]) => args[0] === "provider" && args[1] === "delete", + ), + ).toBe(true); + expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); + expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); }); for (const provider of ["compatible-endpoint", "compatible-anthropic-endpoint"]) { diff --git a/src/lib/actions/inference-set-provider.ts b/src/lib/actions/inference-set-provider.ts index 8bcf01545c..daf4cef5cc 100644 --- a/src/lib/actions/inference-set-provider.ts +++ b/src/lib/actions/inference-set-provider.ts @@ -20,10 +20,31 @@ import { openshellReportsProviderNotFound, } from "./inference-set-error"; import type { InferenceSetProviderBinding } from "./inference-set-route-containment"; +import type { + SandboxInferenceInvocationInput, + SandboxInferenceInvocationResult, +} from "./sandbox/inference-invocation-probe"; export type { RuntimeProviderBundleRegistry }; export { RuntimeProviderSelectionError }; +export type InferenceSetSandboxRouteProbe = ( + input: SandboxInferenceInvocationInput, +) => SandboxInferenceInvocationResult; + +export function probeInferenceSetSandboxRoute( + input: SandboxInferenceInvocationInput, +): SandboxInferenceInvocationResult { + const probe: typeof import("./sandbox/inference-invocation-probe") = require( + "./sandbox/inference-invocation-probe", + ); + return probe.probeSandboxInferenceInvocation( + input, + {}, + probe.READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, + ); +} + export function requireInferenceSetRuntimeAuthority( entry: SandboxEntry, providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, diff --git a/src/lib/actions/inference-set.test-support.ts b/src/lib/actions/inference-set.test-support.ts index 45336d2029..94cb2b679e 100644 --- a/src/lib/actions/inference-set.test-support.ts +++ b/src/lib/actions/inference-set.test-support.ts @@ -135,6 +135,7 @@ export function createDeps(options: { resolveCredentialValue?: InferenceSetDeps["resolveCredentialValue"]; ensureHttpsPinRuntimeAdapter?: EnsureHttpsPinRuntimeAdapterFn; revokeHttpsPinRuntimeAdapterRoute?: InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"]; + probeSandboxRoute?: InferenceSetDeps["probeSandboxRoute"]; updateSandbox?: InferenceSetDeps["updateSandbox"]; restartSandboxGateway?: InferenceSetDeps["restartSandboxGateway"]; seedHermesDashboardConfigResult?: "converged" | "absent" | "failed"; @@ -158,6 +159,7 @@ export function createDeps(options: { resolveCredentialValue: ReturnType; ensureHttpsPinRuntimeAdapter: ReturnType; revokeHttpsPinRuntimeAdapterRoute: ReturnType; + probeSandboxRoute: ReturnType; restartSandboxGateway: ReturnType; withGatewayRouteMutationLock: ReturnType; }; @@ -218,6 +220,7 @@ export function createDeps(options: { revokeHttpsPinRuntimeAdapterRoute: vi.fn( options.revokeHttpsPinRuntimeAdapterRoute ?? (async () => true), ), + probeSandboxRoute: vi.fn(options.probeSandboxRoute ?? (() => ({ ok: true }) as const)), restartSandboxGateway: vi.fn( options.restartSandboxGateway ?? ((): ReturnType => ({ @@ -262,6 +265,7 @@ export function createDeps(options: { calls.ensureHttpsPinRuntimeAdapter as unknown as EnsureHttpsPinRuntimeAdapterFn, revokeHttpsPinRuntimeAdapterRoute: calls.revokeHttpsPinRuntimeAdapterRoute as InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"], + probeSandboxRoute: calls.probeSandboxRoute as InferenceSetDeps["probeSandboxRoute"], withGatewayRouteMutationLock: calls.withGatewayRouteMutationLock as InferenceSetDeps["withGatewayRouteMutationLock"], restartSandboxGateway: calls.restartSandboxGateway, diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index ceb1d71434..4595a3b01b 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -71,7 +71,9 @@ import { readPreviousOpenClawInferenceApi, } from "./inference-set-gateway-restart"; import { + type InferenceSetSandboxRouteProbe, prepareInferenceSetProviderBinding, + probeInferenceSetSandboxRoute, type RuntimeProviderBundleRegistry, RuntimeProviderSelectionError, requireInferenceSetRuntimeAuthority, @@ -164,6 +166,7 @@ export interface InferenceSetDeps extends InferenceGatewayRestartDeps { resolveCredentialValue: (credentialEnv: string) => string; ensureHttpsPinRuntimeAdapter: EnsureHttpsPinRuntimeAdapterFn; revokeHttpsPinRuntimeAdapterRoute: (routeId: string) => Promise; + probeSandboxRoute: InferenceSetSandboxRouteProbe; withGatewayRouteMutationLock: typeof withGatewayRouteMutationLock; } @@ -269,6 +272,7 @@ function defaultDeps(): InferenceSetDeps { resolveCredentialValue: (credentialEnv) => process.env[credentialEnv] ?? "", ensureHttpsPinRuntimeAdapter, revokeHttpsPinRuntimeAdapterRoute, + probeSandboxRoute: probeInferenceSetSandboxRoute, withGatewayRouteMutationLock, restartSandboxGateway: defaultInferenceGatewayRestart, isSandboxConfigMutable: (sandboxName) => { @@ -937,12 +941,13 @@ async function runInferenceSetWithoutHostLock( // verify. Only a genuinely-unreachable host stack hard-fails here, before the // route is touched. let effectiveNoVerify = options.noVerify === true; + const probeDirectSandboxBridge = isSandboxBridgeProviderBinding(directProviderBinding); // Adapter routes and explicit custom routes on NemoClaw's sandbox bridge // resolve only from inside the sandbox network. The host-side OpenShell // verifier cannot resolve host.openshell.internal, so its result would be a - // guaranteed false negative. Endpoint validation above remains the trust - // boundary; the live sandbox request verifies actual route reachability. - if (isSandboxBridgeProviderBinding(httpsPinProviderBinding ?? directProviderBinding)) { + // guaranteed false negative. HTTPS-pin adapters retain their local-health + // verification; direct bridge routes are probed from the sandbox below. + if (httpsPinProviderBinding || probeDirectSandboxBridge) { effectiveNoVerify = true; } if (deps.isLocalInferenceProvider(provider)) { @@ -1001,11 +1006,43 @@ async function runInferenceSetWithoutHostLock( assertReasoningEffortRoute(reasoningEffortRequest, provider, preMutationInferenceApi); const previousProvider = typeof entry.provider === "string" ? entry.provider.trim() : ""; const previousModel = typeof entry.model === "string" ? entry.model.trim() : ""; + if (probeDirectSandboxBridge && (!previousProvider || !previousModel)) { + throw new InferenceSetError( + `Cannot verify the sandbox-only provider route because sandbox '${sandboxName}' does not record ` + + "the previous provider and model needed to restore its OpenShell inference selection.", + 2, + ); + } let appliedProvider = false; let appliedInferenceSelection = false; let restoredSelectionAfterProviderFailure = false; let providerMutation: ReturnType | null = null; + const restorePreviousInferenceSelection = (): string | null => { + let restoreResult: CaptureOpenshellResult; + try { + restoreResult = deps.captureOpenshell( + openshellInferenceSetArgs({ + gatewayName: preparedRoute.gatewayName, + provider: previousProvider, + model: previousModel, + noVerify: true, + }), + { + ignoreError: true, + includeStreams: true, + maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, + }, + ); + } catch { + return "the restore command could not be invoked"; + } + if (restoreResult.status !== 0) { + return `the restore command exited with status ${restoreResult.status ?? "unknown"}`; + } + appliedInferenceSelection = false; + return null; + }; try { const providerBinding = httpsPinProviderBinding ?? directProviderBinding; if (providerBinding) { @@ -1086,28 +1123,15 @@ async function runInferenceSetWithoutHostLock( providerError instanceof Error ? providerError.message : String(providerError); const providerExitCode = providerError instanceof InferenceSetError ? providerError.exitCode : 1; - const restoreResult = deps.captureOpenshell( - openshellInferenceSetArgs({ - gatewayName: preparedRoute.gatewayName, - provider: previousProvider, - model: previousModel, - noVerify: true, - }), - { - ignoreError: true, - includeStreams: true, - maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, - }, - ); - if (restoreResult.status !== 0) { + const restoreFailure = restorePreviousInferenceSelection(); + if (restoreFailure) { throw new InferenceSetError( `${providerDetail}\n Failed to restore the previous OpenShell inference selection ` + - `'${previousProvider}' / '${previousModel}' (status ${restoreResult.status ?? "unknown"}). ` + + `'${previousProvider}' / '${previousModel}': ${restoreFailure}. ` + `The live selection and provider binding may be split; re-run onboarding before using this route.`, providerExitCode, ); } - appliedInferenceSelection = false; restoredSelectionAfterProviderFailure = true; throw new InferenceSetError( `${providerDetail}\n The previous OpenShell inference selection was restored to ` + @@ -1118,6 +1142,38 @@ async function runInferenceSetWithoutHostLock( } } + if (probeDirectSandboxBridge) { + let probe: ReturnType; + try { + probe = deps.probeSandboxRoute({ + sandboxName, + provider, + model, + preferredInferenceApi: preMutationInferenceApi, + }); + } catch { + probe = { + ok: false, + detail: "sandbox inference invocation probe was unavailable", + httpStatus: null, + }; + } + if (!probe.ok) { + const restoreFailure = restorePreviousInferenceSelection(); + if (restoreFailure) { + throw new InferenceSetError( + `Sandbox-side verification rejected provider '${provider}' / '${model}': ${probe.detail}. ` + + `Failed to restore the previous OpenShell inference selection '${previousProvider}' / ` + + `'${previousModel}': ${restoreFailure}. Re-run onboarding before using this route.`, + ); + } + throw new InferenceSetError( + `Sandbox-side verification rejected provider '${provider}' / '${model}': ${probe.detail}. ` + + `The previous OpenShell inference selection was restored to '${previousProvider}' / '${previousModel}'.`, + ); + } + } + // Write minimal registry state before any sandbox-facing config read so the // gateway and registry cannot split if the in-sandbox layer is unavailable. const registryFields = (preferredInferenceApi: string | null) => diff --git a/test/e2e/live/launch-agent-turn.ts b/test/e2e/live/launch-agent-turn.ts index 542bfeb0e9..dd5ecad96b 100644 --- a/test/e2e/live/launch-agent-turn.ts +++ b/test/e2e/live/launch-agent-turn.ts @@ -7,6 +7,68 @@ import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +// The live driver points only the `nemoclaw launch` process at this shim. The +// shim passes every OpenShell call through unchanged except the exact TTY exec +// that starts OpenClaw, where it attaches the non-secret run identity through +// OpenShell's supported request environment. The TUI inherits that identity, +// allowing readiness to bind to this launch instead of a process-table peer. +export const OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT = String.raw`#!/usr/bin/env node +const childProcess = require("node:child_process"); +const fs = require("node:fs"); + +const argv = process.argv.slice(2); +const realOpenShell = process.env.NEMOCLAW_LAUNCH_REAL_OPENSHELL; +const runId = process.env.NEMOCLAW_LAUNCH_RUN_ID; +const sandboxName = process.env.NEMOCLAW_LAUNCH_SANDBOX; +const interceptPath = process.env.NEMOCLAW_LAUNCH_INTERCEPT_PATH; + +function fail(reason) { + process.stderr.write(JSON.stringify({ reason }) + "\n"); + process.exit(73); +} + +function run(nextArgv) { + const result = childProcess.spawnSync(realOpenShell, nextArgv, { stdio: "inherit" }); + if (result.error || result.status === null) fail("openshell_shim_invocation_failed"); + process.exit(result.status); +} + +function arraysEqual(left, right) { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +if (!realOpenShell || !realOpenShell.startsWith("/")) fail("openshell_shim_authority_invalid"); +if (!/^[0-9a-f]{32}$/.test(runId || "")) fail("openshell_shim_run_id_invalid"); +if (!interceptPath || !interceptPath.startsWith("/")) fail("openshell_shim_path_invalid"); + +const separator = argv.indexOf("--"); +const remoteArgv = separator === -1 ? [] : argv.slice(separator + 1); +const expectedTail = ["bash", "-lc", "openclaw tui"]; +let optionIndex = 4; +if (argv[optionIndex] === "-g") optionIndex += 2; +const launchLike = + argv[0] === "sandbox" && + argv[1] === "exec" && + argv[2] === "--name" && + argv[3] === sandboxName && + arraysEqual(argv.slice(optionIndex, separator), ["--tty", "--timeout", "0"]) && + remoteArgv.length >= expectedTail.length && + expectedTail.every((value, index) => value === remoteArgv.at(index - expectedTail.length)); + +if (!launchLike) run(argv); +try { + fs.writeFileSync(interceptPath, runId + "\n", { flag: "wx", mode: 0o600 }); +} catch { + fail("openshell_launch_intercept_duplicate"); +} +run([ + ...argv.slice(0, separator), + "--env", + "NEMOCLAW_LAUNCH_RUN_ID=" + runId, + ...argv.slice(separator), +]); +`; + // OpenClaw owns the JSONL session store and does not expose a structured // result from `nemoclaw launch`. This verifier records an in-sandbox baseline, // then qualifies only complete user and assistant records appended after that @@ -16,7 +78,7 @@ const crypto = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); -const [mode, sessionRoot, baselinePath, expectedTurnsText] = process.argv.slice(1); +const [mode, sessionRoot, baselinePath, expectedTurnsText, runId] = process.argv.slice(1); function finish(exitCode, reason, detail = {}) { if (reason) process.stderr.write(JSON.stringify({ reason, ...detail }) + "\n"); @@ -43,7 +105,7 @@ function sessionFileNames() { } } -function openClawTuiProcessIds() { +function launchOwnedOpenClawTuiProcessIds() { let names; try { names = fs.readdirSync("/proc"); @@ -65,19 +127,27 @@ function openClawTuiProcessIds() { } if (!args.includes("tui")) continue; if (!args.some((arg) => ["openclaw", "openclaw.mjs"].includes(path.basename(arg)))) continue; - pids.push(name); + let environment; + try { + environment = fs.readFileSync(path.join("/proc", name, "environ"), "utf8").split("\0"); + } catch (error) { + if (error && ["ENOENT", "ESRCH"].includes(error.code)) continue; + finish(2, "tui_environment_unavailable"); + } + if (environment.includes("NEMOCLAW_LAUNCH_RUN_ID=" + runId)) pids.push(name); } return pids; } // The terminal line discipline safely queues a complete submitted line even // while a canonical-mode TUI is still installing its reader. Raw mode is a UI -// implementation detail, so the stable readiness boundary is the one matching -// OpenClaw TUI process owning the launch PTY on standard input. +// implementation detail. Readiness therefore requires the exact run identity +// injected into this launch's remote exec and a PTY on that process's fd 0. function qualifyTuiInputPty() { - const pids = openClawTuiProcessIds(); + if (!/^[0-9a-f]{32}$/.test(runId || "")) finish(2, "launch_run_id_invalid"); + const pids = launchOwnedOpenClawTuiProcessIds(); if (pids.length === 0) finish(1); - if (pids.length > 1) finish(2, "multiple_tui_processes"); + if (pids.length > 1) finish(2, "multiple_launch_tui_processes"); let ttyPath; try { ttyPath = fs.realpathSync(path.join("/proc", pids[0], "fd", "0")); @@ -238,6 +308,9 @@ capture="$session_dir/terminal.log" driver_error="$session_dir/pty-driver.err" evidence_error="$session_dir/session-evidence.err" input="$session_dir/input" +input_submitted_marker="$session_dir/input-submitted" +openshell_shim="$session_dir/openshell-launch-shim" +intercept_path="$session_dir/launch-intercept" baseline_path="/tmp/nemoclaw-launch-session-$NEMOCLAW_LAUNCH_RUN_ID.json" session_pid="" session_deadline="" @@ -316,7 +389,8 @@ session_evidence() { "$mode" \ "$NEMOCLAW_LAUNCH_SESSION_ROOT" \ "$baseline_path" \ - "$expected_turns" + "$expected_turns" \ + "$NEMOCLAW_LAUNCH_RUN_ID" } wait_for_turn_count() { @@ -362,6 +436,8 @@ if ! session_evidence baseline >/dev/null 2>"$evidence_error"; then fail_launch_session "launch could not record the structured session baseline" fi +printf '%s' "$NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT" >"$openshell_shim" +chmod 700 "$openshell_shim" mkfifo -m 600 "$input" if [[ -n "$NEMOCLAW_LAUNCH_ENTRYPOINT" ]]; then printf -v launch_command '%q %q %q %q' \ @@ -372,6 +448,10 @@ else "$NEMOCLAW_LAUNCH_COMMAND" launch "$NEMOCLAW_LAUNCH_SANDBOX" fi +NEMOCLAW_LAUNCH_INPUT_SUBMITTED_MARKER="$input_submitted_marker" \ +NEMOCLAW_LAUNCH_INTERCEPT_PATH="$intercept_path" \ +NEMOCLAW_LAUNCH_REAL_OPENSHELL="$NEMOCLAW_OPENSHELL_COMMAND" \ +NEMOCLAW_OPENSHELL_BIN="$openshell_shim" \ timeout --kill-after=5s 250s \ script --quiet --return --flush --command "$launch_command" "$capture" \ <"$input" >/dev/null 2>"$driver_error" & @@ -399,6 +479,7 @@ wait_for_tui_input_pty if ! printf '%s\r' "$NEMOCLAW_LAUNCH_FIRST_INPUT" >&3; then fail_launch_session "launch exited before the first PTY input was submitted" fi +: >"$input_submitted_marker" wait_for_turn_count 1 if ! printf '%s\r' "$NEMOCLAW_LAUNCH_SECOND_INPUT" >&3; then fail_launch_session "launch exited before the second PTY input was submitted" @@ -467,6 +548,9 @@ export async function runOpenClawLaunchSession( if (process.platform !== "linux") { throw new Error("launch session coverage requires the Linux util-linux PTY driver"); } + if (!options.host.openshellCommandPath.startsWith("/")) { + throw new Error("launch session coverage requires an absolute OpenShell command path"); + } const inputs = uniqueTurnInputs(); const result = await options.host.command("bash", ["-lc", LAUNCH_TURN_SCRIPT], { artifactName: options.artifactName, @@ -476,6 +560,7 @@ export async function runOpenClawLaunchSession( NEMOCLAW_LAUNCH_ENTRYPOINT: options.cliEntrypoint ?? "", NEMOCLAW_LAUNCH_EXIT_COMMAND: options.exitCommand ?? "", NEMOCLAW_LAUNCH_FIRST_INPUT: inputs.first, + NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT: OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, NEMOCLAW_LAUNCH_RUN_ID: randomUUID().replaceAll("-", ""), NEMOCLAW_LAUNCH_SANDBOX: options.sandboxName, NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: "230", diff --git a/test/e2e/support/launch-agent-turn.test.ts b/test/e2e/support/launch-agent-turn.test.ts index 3fbcb03a62..52a7cd0579 100644 --- a/test/e2e/support/launch-agent-turn.test.ts +++ b/test/e2e/support/launch-agent-turn.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; import { appendFileSync, chmodSync, @@ -14,11 +15,12 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { basename, join } from "node:path"; +import { join } from "node:path"; import { expect, it } from "vitest"; import { LAUNCH_TURN_SCRIPT, + OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, OPENCLAW_SESSION_EVIDENCE_SCRIPT, runOpenClawLaunchReadinessLeaseTurns, } from "../live/launch-agent-turn.ts"; @@ -32,6 +34,7 @@ type FixtureMode = | "delayed-recording" | "invalid-order" | "late-extra" + | "mismatched-tui-pty" | "multiple-tui-processes" | "nonzero" | "nonzero-cleanup-failure" @@ -145,7 +148,10 @@ function runLaunchSessionFixture(mode: FixtureMode, terminalCopy: "absent" | "an const tuiStdinRetryMarker = join(fixtureRoot, "tui-stdin-retry"); const tuiStdinUnavailableMarker = join(fixtureRoot, "tui-stdin-unavailable"); const ttyMarker = join(fixtureRoot, "tty-observed"); - const runId = basename(fixtureRoot).replaceAll(/[^a-zA-Z0-9]/gu, ""); + const launchTtyMarker = join(fixtureRoot, "launch-tty"); + const unrelatedTtyMarker = join(fixtureRoot, "unrelated-tty"); + const readerOrderMarker = join(fixtureRoot, "reader-after-submission"); + const runId = randomUUID().replaceAll("-", ""); const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; mkdirSync(sessionRoot); mkdirSync(tuiInputMarkerRoot); @@ -160,7 +166,37 @@ const childProcess = require("node:child_process"); const mode = process.env.NEMOCLAW_FIXTURE_MODE; if (process.argv[2] !== "tui") { - if (mode === "transient-tui-stdin") { + if (mode === "mismatched-tui-pty") { + fs.writeFileSync( + process.env.NEMOCLAW_FIXTURE_LAUNCH_TTY_MARKER, + fs.realpathSync("/proc/self/fd/0"), + ); + const unrelated = childProcess.spawn( + "script", + [ + "--quiet", + "--return", + "--command", + '"$NEMOCLAW_FIXTURE_NODE" "$NEMOCLAW_FIXTURE_SCRIPT" tui unrelated', + "/dev/null", + ], + { + env: { + ...process.env, + NEMOCLAW_LAUNCH_RUN_ID: "00000000000000000000000000000000", + }, + stdio: "ignore", + }, + ); + const stopUnrelated = () => { + try { unrelated.kill("SIGTERM"); } catch {} + setTimeout(() => process.exit(0), 100); + }; + for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"]) { + process.once(signal, stopUnrelated); + } + unrelated.once("exit", (status) => process.exit(status ?? 66)); + } else if (mode === "transient-tui-stdin") { const transient = childProcess.spawn( process.execPath, [__filename, "tui", "stdin-unavailable"], @@ -225,6 +261,13 @@ if (process.argv[2] !== "tui") { if (process.argv[2] === "tui") (async () => { if (!process.stdin.isTTY || !process.stdout.isTTY) process.exit(64); fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_TTY_MARKER, ""); + if (process.argv[3] === "unrelated") { + fs.appendFileSync(process.env.NEMOCLAW_FIXTURE_TUI_PIDS, process.pid + "\n"); + fs.writeFileSync( + process.env.NEMOCLAW_FIXTURE_UNRELATED_TTY_MARKER, + fs.realpathSync("/proc/self/fd/0"), + ); + } const sessionFile = process.env.NEMOCLAW_FIXTURE_SESSION_FILE; const terminalCopy = process.env.NEMOCLAW_FIXTURE_TERMINAL_COPY; const append = (role, content) => fs.appendFileSync( @@ -232,6 +275,7 @@ if (process.argv[2] === "tui") (async () => { JSON.stringify({ message: { content: [{ text: content, type: "text" }], role }, type: "message" }) + "\n", ); if ( + mode === "mismatched-tui-pty" || mode === "multiple-tui-processes" || (mode === "transient-tui-stdin" && process.argv[3] !== "stdin-unavailable") ) { @@ -256,7 +300,10 @@ if (process.argv[2] === "tui") (async () => { process.exit(fs.existsSync(process.env.NEMOCLAW_FIXTURE_TUI_STDIN_RETRY) ? 0 : 68); } if (mode === "delayed-input-reader") { - await new Promise((resolve) => setTimeout(resolve, 1_500)); + while (!fs.existsSync(process.env.NEMOCLAW_LAUNCH_INPUT_SUBMITTED_MARKER)) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_READER_ORDER_MARKER, ""); } const rl = readline.createInterface({ input: process.stdin, output: process.stdout, terminal: true }); const ask = () => new Promise((resolve) => rl.question("", resolve)); @@ -305,7 +352,15 @@ set -euo pipefail if [[ "$NEMOCLAW_FIXTURE_MODE" == *"cleanup-failure" && " $* " == *" rm -f -- "* ]]; then exit 71 fi -while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done +injected_env="" +while [[ "$#" -gt 0 && "$1" != "--" ]]; do + if [[ "$1" == "--env" ]]; then + injected_env="$2" + shift 2 + else + shift + fi +done [[ "$#" -gt 0 ]] shift if [[ "$NEMOCLAW_FIXTURE_MODE" == "transient-tui-stdin" && "$4" == "input-pty" ]]; then @@ -318,7 +373,10 @@ if [[ "$NEMOCLAW_FIXTURE_MODE" == "transient-tui-stdin" && "$4" == "input-pty" ] fi exit "$status" fi -exec "$@" +if [[ -n "$injected_env" ]]; then + exec env -u NEMOCLAW_LAUNCH_RUN_ID "$injected_env" "$@" +fi +exec env -u NEMOCLAW_LAUNCH_RUN_ID "$@" `, ); chmodSync(fakeLaunch, 0o755); @@ -330,6 +388,10 @@ exec "$@" env: { ...process.env, NEMOCLAW_FIXTURE_MODE: mode, + NEMOCLAW_FIXTURE_LAUNCH_TTY_MARKER: launchTtyMarker, + NEMOCLAW_FIXTURE_NODE: process.execPath, + NEMOCLAW_FIXTURE_READER_ORDER_MARKER: readerOrderMarker, + NEMOCLAW_FIXTURE_SCRIPT: fakeLaunch, NEMOCLAW_FIXTURE_SESSION_FILE: join(sessionRoot, "session-a.jsonl"), NEMOCLAW_FIXTURE_TERMINAL_COPY: terminalCopy, NEMOCLAW_FIXTURE_TUI_INPUT_MARKER_ROOT: tuiInputMarkerRoot, @@ -337,13 +399,16 @@ exec "$@" NEMOCLAW_FIXTURE_TUI_STDIN_RETRY: tuiStdinRetryMarker, NEMOCLAW_FIXTURE_TUI_STDIN_UNAVAILABLE: tuiStdinUnavailableMarker, NEMOCLAW_FIXTURE_TTY_MARKER: ttyMarker, + NEMOCLAW_FIXTURE_UNRELATED_TTY_MARKER: unrelatedTtyMarker, NEMOCLAW_LAUNCH_COMMAND: fakeLaunch, NEMOCLAW_LAUNCH_ENTRYPOINT: "", NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", NEMOCLAW_LAUNCH_FIRST_INPUT: "first input", + NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT: OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, NEMOCLAW_LAUNCH_RUN_ID: runId, NEMOCLAW_LAUNCH_SANDBOX: "sandbox", - NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: mode.endsWith("-timeout") ? "2" : "230", + NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: + mode.endsWith("-timeout") || mode === "mismatched-tui-pty" ? "2" : "230", NEMOCLAW_LAUNCH_SECOND_INPUT: "second input", NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, NEMOCLAW_LAUNCH_SESSION_ROOT: sessionRoot, @@ -370,9 +435,14 @@ exec "$@" existsSync(join(tuiInputMarkerRoot, pid)), ), result, + launchTty: existsSync(launchTtyMarker) ? readFileSync(launchTtyMarker, "utf8") : null, + readerInstalledAfterSubmission: existsSync(readerOrderMarker), tuiStdinUnavailableObserved: existsSync(tuiStdinUnavailableMarker), tuiProcessIds, ttyObserved: existsSync(ttyMarker), + unrelatedTty: existsSync(unrelatedTtyMarker) + ? readFileSync(unrelatedTtyMarker, "utf8") + : null, }; } finally { rmSync(fixtureRoot, { force: true, recursive: true }); @@ -483,18 +553,43 @@ it.runIf(process.platform === "linux")( it.runIf(process.platform === "linux")( "queues one canonical PTY line until the OpenClaw TUI installs its input reader (#9160)", () => { - const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( - "delayed-input-reader", - "absent", - ); + const { baselineRemoved, readerInstalledAfterSubmission, result, ttyObserved } = + runLaunchSessionFixture("delayed-input-reader", "absent"); expect(ttyObserved).toBe(true); + expect(readerInstalledAfterSubmission).toBe(true); expect(baselineRemoved).toBe(true); expect(result.signal).toBeNull(); expect(result.status).toBe(0); }, ); +it.runIf(process.platform === "linux")( + "rejects an unrelated OpenClaw TUI on a different PTY before submitting input (#9160)", + () => { + const { + baselineRemoved, + launchTty, + orphanedTuiProcessIds, + recordedTuiInputProcessIds, + result, + unrelatedTty, + } = runLaunchSessionFixture("mismatched-tui-pty", "absent"); + + expect(launchTty).toMatch(/^\/dev\/pts\/\d+$/); + expect(unrelatedTty).toMatch(/^\/dev\/pts\/\d+$/); + expect(unrelatedTty).not.toBe(launchTty); + expect(recordedTuiInputProcessIds).toEqual([]); + expect(orphanedTuiProcessIds).toEqual([]); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "OpenClaw TUI did not attach standard input to the launch PTY before the session deadline", + ); + }, +); + it.runIf(process.platform === "linux")( "retries when a matching OpenClaw TUI process closes standard input (#9160)", () => { @@ -538,7 +633,7 @@ it.runIf(process.platform === "linux")( expect(baselineRemoved).toBe(true); expect(result.signal).toBeNull(); expect(result.status).toBe(1); - expect(result.stderr).toContain('"reason":"multiple_tui_processes"'); + expect(result.stderr).toContain('"reason":"multiple_launch_tui_processes"'); }, ); From bf774e934d79f586dc0788476b16bbbc2df2530b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 16 Aug 2026 04:50:50 -0700 Subject: [PATCH 3/8] docs(inference): clarify rollback recovery Signed-off-by: Prekshi Vyas --- docs/inference/switch-providers.mdx | 3 ++- docs/reference/commands.mdx | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/inference/switch-providers.mdx b/docs/inference/switch-providers.mdx index 001722e700..d1f85db688 100644 --- a/docs/inference/switch-providers.mdx +++ b/docs/inference/switch-providers.mdx @@ -75,7 +75,8 @@ Use `--no-verify` only when OpenShell cannot verify the target provider at switc This flag does not bypass shared-gateway compatibility checks. For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. It then sends one minimal request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens. -If that request fails, NemoClaw restores the previous OpenShell selection and removes a provider that this switch created. +If that request fails, NemoClaw attempts to restore the previous OpenShell selection and remove a provider that this switch created. +If the error reports that rollback could not complete, re-run onboarding before using or retrying the route. Endpoint-shape and shared-gateway compatibility checks still apply. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 9b075a9759..14f7a9c7ba 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -4102,7 +4102,8 @@ Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `open Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential. For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. Before it persists the route in the NemoClaw registry or agent config, the command sends one request from the target sandbox with a 16-token output limit. -If that request fails, the command restores the previous OpenShell selection and removes a provider that this switch created. +If that request fails, the command attempts to restore the previous OpenShell selection and remove a provider that this switch created. +If the error reports that rollback could not complete, re-run onboarding before using or retrying the route. Endpoint-shape and shared-gateway compatibility checks still apply. When switching to `compatible-endpoint` or `compatible-anthropic-endpoint` from a different provider family, pass `--endpoint-url` with the trusted custom provider URL and, except for the Hermes case below, `--inference-api` with its API family so NemoClaw can persist a complete route identity for rebuild and shared-gateway checks. For a Hermes `compatible-anthropic-endpoint` target, `--inference-api` may be omitted because NemoClaw deterministically selects `openai-completions`; an explicit different API family is rejected. From b0864192a5af8403266195ed304a7abb96dd17e8 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 16 Aug 2026 05:09:35 -0700 Subject: [PATCH 4/8] fix(e2e): harden route and PTY failure evidence Signed-off-by: Prekshi Vyas --- .../inference-set-compatible-provider.test.ts | 104 +++++++++++ src/lib/actions/inference-set.ts | 10 +- test/e2e/live/launch-agent-turn.ts | 2 +- test/e2e/support/launch-agent-turn.test.ts | 175 +++++++++++------- 4 files changed, 218 insertions(+), 73 deletions(-) diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index cc998c94dd..aea1cb378e 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -745,6 +745,110 @@ describe("runInferenceSet compatible providers", () => { expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); }); + it("preserves redacted probe diagnostics when restoring the prior route fails", async () => { + const providerCapture = createCompatibleProviderCapture({ + name: "compatible-anthropic-endpoint", + type: "anthropic", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + configKey: "ANTHROPIC_BASE_URL", + initiallyPresent: false, + }); + const inferenceSetResults = [ + null, + { + status: 19, + output: "restore rejected", + stdout: "", + stderr: "restore rejected", + }, + ]; + let inferenceSetCalls = 0; + const captureOpenshell = vi.fn((args: string[]) => { + switch (`${args[0]}:${args[1]}`) { + case "inference:set": + return inferenceSetResults[inferenceSetCalls++] ?? providerCapture(args); + default: + return providerCapture(args); + } + }); + const deps = createDeps({ + config: { agents: { defaults: { model: { primary: "inference/old-model" } } } }, + entry: { + name: "alpha", + agent: "openclaw", + provider: "nvidia-prod", + model: "old-model", + }, + session: baseSession({ provider: "nvidia-prod", model: "old-model" }), + captureOpenshell, + probeSandboxRoute: () => { + throw new Error("sandbox dial failed; NVIDIA_API_KEY=nvapi-secret-value"); + }, + }); + + let failure: unknown; + try { + await runInferenceSet( + { + provider: "compatible-anthropic-endpoint", + model: "mock-anthropic-model", + endpointUrl: "http://host.openshell.internal:18767/", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + inferenceApi: "anthropic-messages", + }, + deps, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(Error); + const failureMessage = (failure as Error).message; + expect(failureMessage).toContain( + "sandbox inference invocation probe was unavailable: sandbox dial failed", + ); + expect(failureMessage).toContain("NVIDIA_API_KEY="); + expect(failureMessage).not.toContain("nvapi-secret-value"); + expect(failureMessage).toMatch( + /Failed to restore the previous OpenShell inference selection.*status 19.*Re-run onboarding/s, + ); + expect( + deps.calls.captureOpenshell.mock.calls + .filter(([args]) => args[0] === "inference" && args[1] === "set") + .map(([args]) => args), + ).toEqual([ + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "compatible-anthropic-endpoint", + "--model", + "mock-anthropic-model", + "--no-verify", + ], + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "nvidia-prod", + "--model", + "old-model", + "--no-verify", + ], + ]); + expect( + deps.calls.captureOpenshell.mock.calls.some( + ([args]) => args[0] === "provider" && args[1] === "delete", + ), + ).toBe(false); + expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); + expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); + }); + for (const provider of ["compatible-endpoint", "compatible-anthropic-endpoint"]) { it.each([ ["loopback", "http://127.0.0.1:8000/v1", "93.184.216.34"], diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index 4595a3b01b..c6fcf0d0f1 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -1151,10 +1151,16 @@ async function runInferenceSetWithoutHostLock( model, preferredInferenceApi: preMutationInferenceApi, }); - } catch { + } catch (probeError) { + const probeFailureDetail = + probeError instanceof Error && probeError.message + ? (onboardSession.redactSensitiveText(probeError.message)?.trim() ?? "") + : ""; probe = { ok: false, - detail: "sandbox inference invocation probe was unavailable", + detail: probeFailureDetail + ? `sandbox inference invocation probe was unavailable: ${probeFailureDetail}` + : "sandbox inference invocation probe was unavailable", httpStatus: null, }; } diff --git a/test/e2e/live/launch-agent-turn.ts b/test/e2e/live/launch-agent-turn.ts index dd5ecad96b..da94e40e5f 100644 --- a/test/e2e/live/launch-agent-turn.ts +++ b/test/e2e/live/launch-agent-turn.ts @@ -131,7 +131,7 @@ function launchOwnedOpenClawTuiProcessIds() { try { environment = fs.readFileSync(path.join("/proc", name, "environ"), "utf8").split("\0"); } catch (error) { - if (error && ["ENOENT", "ESRCH"].includes(error.code)) continue; + if (error && ["EACCES", "ENOENT", "ESRCH"].includes(error.code)) continue; finish(2, "tui_environment_unavailable"); } if (environment.includes("NEMOCLAW_LAUNCH_RUN_ID=" + runId)) pids.push(name); diff --git a/test/e2e/support/launch-agent-turn.test.ts b/test/e2e/support/launch-agent-turn.test.ts index 52a7cd0579..3496a7a6d8 100644 --- a/test/e2e/support/launch-agent-turn.test.ts +++ b/test/e2e/support/launch-agent-turn.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { appendFileSync, @@ -153,9 +153,41 @@ function runLaunchSessionFixture(mode: FixtureMode, terminalCopy: "absent" | "an const readerOrderMarker = join(fixtureRoot, "reader-after-submission"); const runId = randomUUID().replaceAll("-", ""); const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; + let unrelatedScriptProcess: ReturnType | null = null; mkdirSync(sessionRoot); mkdirSync(tuiInputMarkerRoot); + const recordedTuiProcessIds = (): string[] => + existsSync(tuiPidsPath) + ? readFileSync(tuiPidsPath, "utf8").trim().split("\n").filter(Boolean) + : []; + const processIsRunning = (pid: string): boolean => { + try { + const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); + return stat.slice(stat.lastIndexOf(")") + 2, stat.lastIndexOf(")") + 3) !== "Z"; + } catch { + return false; + } + }; + const stopUnrelatedTui = (): void => { + const stopAndWait = (processIds: string[]): void => { + for (const signal of ["SIGTERM", "SIGKILL"] as const) { + for (const pid of processIds) { + try { + process.kill(Number(pid), signal); + } catch {} + } + const exitDeadline = Date.now() + 1_000; + while (processIds.some(processIsRunning) && Date.now() < exitDeadline) { + Atomics.wait(PROCESS_EXIT_WAIT, 0, 0, 25); + } + } + }; + stopAndWait(recordedTuiProcessIds()); + stopAndWait(unrelatedScriptProcess?.pid ? [String(unrelatedScriptProcess.pid)] : []); + unrelatedScriptProcess = null; + }; + try { writeFileSync( fakeLaunch, @@ -171,31 +203,8 @@ if (process.argv[2] !== "tui") { process.env.NEMOCLAW_FIXTURE_LAUNCH_TTY_MARKER, fs.realpathSync("/proc/self/fd/0"), ); - const unrelated = childProcess.spawn( - "script", - [ - "--quiet", - "--return", - "--command", - '"$NEMOCLAW_FIXTURE_NODE" "$NEMOCLAW_FIXTURE_SCRIPT" tui unrelated', - "/dev/null", - ], - { - env: { - ...process.env, - NEMOCLAW_LAUNCH_RUN_ID: "00000000000000000000000000000000", - }, - stdio: "ignore", - }, - ); - const stopUnrelated = () => { - try { unrelated.kill("SIGTERM"); } catch {} - setTimeout(() => process.exit(0), 100); - }; - for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"]) { - process.once(signal, stopUnrelated); - } - unrelated.once("exit", (status) => process.exit(status ?? 66)); + process.stdin.resume(); + process.stdin.once("end", () => process.exit(0)); } else if (mode === "transient-tui-stdin") { const transient = childProcess.spawn( process.execPath, @@ -382,55 +391,80 @@ exec env -u NEMOCLAW_LAUNCH_RUN_ID "$@" chmodSync(fakeLaunch, 0o755); chmodSync(fakeOpenshell, 0o755); + const fixtureEnv: NodeJS.ProcessEnv = { + ...process.env, + NEMOCLAW_FIXTURE_MODE: mode, + NEMOCLAW_FIXTURE_LAUNCH_TTY_MARKER: launchTtyMarker, + NEMOCLAW_FIXTURE_NODE: process.execPath, + NEMOCLAW_FIXTURE_READER_ORDER_MARKER: readerOrderMarker, + NEMOCLAW_FIXTURE_SCRIPT: fakeLaunch, + NEMOCLAW_FIXTURE_SESSION_FILE: join(sessionRoot, "session-a.jsonl"), + NEMOCLAW_FIXTURE_TERMINAL_COPY: terminalCopy, + NEMOCLAW_FIXTURE_TUI_INPUT_MARKER_ROOT: tuiInputMarkerRoot, + NEMOCLAW_FIXTURE_TUI_PIDS: tuiPidsPath, + NEMOCLAW_FIXTURE_TUI_STDIN_RETRY: tuiStdinRetryMarker, + NEMOCLAW_FIXTURE_TUI_STDIN_UNAVAILABLE: tuiStdinUnavailableMarker, + NEMOCLAW_FIXTURE_TTY_MARKER: ttyMarker, + NEMOCLAW_FIXTURE_UNRELATED_TTY_MARKER: unrelatedTtyMarker, + NEMOCLAW_LAUNCH_COMMAND: fakeLaunch, + NEMOCLAW_LAUNCH_ENTRYPOINT: "", + NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", + NEMOCLAW_LAUNCH_FIRST_INPUT: "first input", + NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT: OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, + NEMOCLAW_LAUNCH_RUN_ID: runId, + NEMOCLAW_LAUNCH_SANDBOX: "sandbox", + NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: + mode.endsWith("-timeout") || mode === "mismatched-tui-pty" ? "2" : "230", + NEMOCLAW_LAUNCH_SECOND_INPUT: "second input", + NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, + NEMOCLAW_LAUNCH_SESSION_ROOT: sessionRoot, + NEMOCLAW_OPENSHELL_COMMAND: fakeOpenshell, + TERM: "xterm-256color", + }; + const startUnrelatedTui = (): void => { + unrelatedScriptProcess = spawn( + "script", + [ + "--quiet", + "--return", + "--command", + '"$NEMOCLAW_FIXTURE_NODE" "$NEMOCLAW_FIXTURE_SCRIPT" tui unrelated', + "/dev/null", + ], + { + env: { + ...fixtureEnv, + NEMOCLAW_LAUNCH_RUN_ID: "00000000000000000000000000000000", + }, + stdio: "ignore", + }, + ); + const unrelatedReadyDeadline = Date.now() + 5_000; + while ( + !existsSync(unrelatedTtyMarker) && + unrelatedScriptProcess.pid && + existsSync(`/proc/${unrelatedScriptProcess.pid}`) && + Date.now() < unrelatedReadyDeadline + ) { + Atomics.wait(PROCESS_EXIT_WAIT, 0, 0, 25); + } + expect(existsSync(unrelatedTtyMarker), "unrelated TUI fixture PTY identity").toBe(true); + }; + const setupFixture = mode === "mismatched-tui-pty" ? startUnrelatedTui : () => undefined; + setupFixture(); + const result = spawnSync("bash", ["-c", LAUNCH_TURN_SCRIPT], { encoding: "utf8", killSignal: "SIGKILL", - env: { - ...process.env, - NEMOCLAW_FIXTURE_MODE: mode, - NEMOCLAW_FIXTURE_LAUNCH_TTY_MARKER: launchTtyMarker, - NEMOCLAW_FIXTURE_NODE: process.execPath, - NEMOCLAW_FIXTURE_READER_ORDER_MARKER: readerOrderMarker, - NEMOCLAW_FIXTURE_SCRIPT: fakeLaunch, - NEMOCLAW_FIXTURE_SESSION_FILE: join(sessionRoot, "session-a.jsonl"), - NEMOCLAW_FIXTURE_TERMINAL_COPY: terminalCopy, - NEMOCLAW_FIXTURE_TUI_INPUT_MARKER_ROOT: tuiInputMarkerRoot, - NEMOCLAW_FIXTURE_TUI_PIDS: tuiPidsPath, - NEMOCLAW_FIXTURE_TUI_STDIN_RETRY: tuiStdinRetryMarker, - NEMOCLAW_FIXTURE_TUI_STDIN_UNAVAILABLE: tuiStdinUnavailableMarker, - NEMOCLAW_FIXTURE_TTY_MARKER: ttyMarker, - NEMOCLAW_FIXTURE_UNRELATED_TTY_MARKER: unrelatedTtyMarker, - NEMOCLAW_LAUNCH_COMMAND: fakeLaunch, - NEMOCLAW_LAUNCH_ENTRYPOINT: "", - NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", - NEMOCLAW_LAUNCH_FIRST_INPUT: "first input", - NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT: OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, - NEMOCLAW_LAUNCH_RUN_ID: runId, - NEMOCLAW_LAUNCH_SANDBOX: "sandbox", - NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: - mode.endsWith("-timeout") || mode === "mismatched-tui-pty" ? "2" : "230", - NEMOCLAW_LAUNCH_SECOND_INPUT: "second input", - NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, - NEMOCLAW_LAUNCH_SESSION_ROOT: sessionRoot, - NEMOCLAW_OPENSHELL_COMMAND: fakeOpenshell, - TERM: "xterm-256color", - }, + env: fixtureEnv, timeout: 15_000, }); - const tuiProcessIds = existsSync(tuiPidsPath) - ? readFileSync(tuiPidsPath, "utf8").trim().split("\n").filter(Boolean) - : []; - const processExitDeadline = Date.now() + 1_000; - while ( - tuiProcessIds.some((pid) => existsSync(`/proc/${pid}`)) && - Date.now() < processExitDeadline - ) { - Atomics.wait(PROCESS_EXIT_WAIT, 0, 0, 25); - } + const tuiProcessIds = recordedTuiProcessIds(); + stopUnrelatedTui(); return { baselineRemoved: !existsSync(baselinePath), - orphanedTuiProcessIds: tuiProcessIds.filter((pid) => existsSync(`/proc/${pid}`)), + orphanedTuiProcessIds: tuiProcessIds.filter(processIsRunning), recordedTuiInputProcessIds: tuiProcessIds.filter((pid) => existsSync(join(tuiInputMarkerRoot, pid)), ), @@ -445,6 +479,7 @@ exec env -u NEMOCLAW_LAUNCH_RUN_ID "$@" : null, }; } finally { + stopUnrelatedTui(); rmSync(fixtureRoot, { force: true, recursive: true }); rmSync(baselinePath, { force: true }); } @@ -754,7 +789,7 @@ it.runIf(process.platform === "linux")( calls.push({ command, args, env: options?.env }); return { exitCode: 0, signal: null, stdout: "", stderr: "" }; }, - openshellCommandPath: "openshell", + openshellCommandPath: "/usr/bin/openshell", }; await runOpenClawLaunchReadinessLeaseTurns({ @@ -787,8 +822,8 @@ it.runIf(process.platform === "linux")( "/exit", ]); expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_OPENSHELL_COMMAND)).toEqual([ - "openshell", - "openshell", + "/usr/bin/openshell", + "/usr/bin/openshell", ]); for (const call of calls.slice(1)) { expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_EXPECTED_REPLY"); From f69668ccbdeef4641aae19ffda9224dce922a1e0 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 16 Aug 2026 05:15:30 -0700 Subject: [PATCH 5/8] docs(inference): scope direct bridge verification Signed-off-by: Prekshi Vyas --- docs/inference/switch-providers.mdx | 2 +- docs/reference/commands.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/inference/switch-providers.mdx b/docs/inference/switch-providers.mdx index d1f85db688..d0f793761e 100644 --- a/docs/inference/switch-providers.mdx +++ b/docs/inference/switch-providers.mdx @@ -73,7 +73,7 @@ Run the rebuild before relying on the running agent. Use `--no-verify` only when OpenShell cannot verify the target provider at switch time and you have already confirmed its provider and credential. This flag does not bypass shared-gateway compatibility checks. -For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +When you explicitly supply a direct compatible-provider endpoint at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. It then sends one minimal request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens. If that request fails, NemoClaw attempts to restore the previous OpenShell selection and remove a provider that this switch created. If the error reports that rollback could not complete, re-run onboarding before using or retrying the route. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 14f7a9c7ba..e7954fedf5 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -4100,7 +4100,7 @@ If the in-sandbox config sync fails, NemoClaw keeps the gateway and registry ali Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `hermes-provider`, `ollama-local`, and `vllm-local`. Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential. -For a validated compatible-provider binding at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +When you explicitly supply a direct compatible-provider endpoint at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. Before it persists the route in the NemoClaw registry or agent config, the command sends one request from the target sandbox with a 16-token output limit. If that request fails, the command attempts to restore the previous OpenShell selection and remove a provider that this switch created. If the error reports that rollback could not complete, re-run onboarding before using or retrying the route. From 4de142840ae7cb8b0d81ff963e770031e6ec839a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 16 Aug 2026 05:22:53 -0700 Subject: [PATCH 6/8] test(inference): cover thrown probe rollback Signed-off-by: Prekshi Vyas --- .../inference-set-compatible-provider.test.ts | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index aea1cb378e..85b343c05f 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -668,7 +668,24 @@ describe("runInferenceSet compatible providers", () => { ); }); - it("restores the prior route when sandbox-only provider verification fails", async () => { + it.each([ + [ + "returns a rejection", + () => ({ + ok: false, + detail: "sandbox inference invocation probe exited with status 7", + httpStatus: null, + }), + /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s, + ], + [ + "throws", + () => { + throw new Error("sandbox dial failed"); + }, + /sandbox inference invocation probe was unavailable: sandbox dial failed.*previous OpenShell inference selection was restored/s, + ], + ])("restores the prior route when sandbox-only provider verification %s", async (_failureMode, probeSandboxRoute, expectedError) => { const captureOpenshell = createCompatibleProviderCapture({ name: "compatible-anthropic-endpoint", type: "anthropic", @@ -686,11 +703,7 @@ describe("runInferenceSet compatible providers", () => { }, session: baseSession({ provider: "nvidia-prod", model: "old-model" }), captureOpenshell, - probeSandboxRoute: () => ({ - ok: false, - detail: "sandbox inference invocation probe exited with status 7", - httpStatus: null, - }), + probeSandboxRoute, }); await expect( @@ -704,9 +717,7 @@ describe("runInferenceSet compatible providers", () => { }, deps, ), - ).rejects.toThrow( - /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s, - ); + ).rejects.toThrow(expectedError); expect( captureOpenshell.mock.calls From 5bbdc7d52af81f48dfb7fb6e5c72deccbd30caaf Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 16 Aug 2026 08:15:01 -0700 Subject: [PATCH 7/8] test(e2e): leave PTY repair to dedicated PR Signed-off-by: Prekshi Vyas --- test/e2e/live/launch-agent-turn.ts | 126 ++-------- test/e2e/support/launch-agent-turn.test.ts | 260 ++++++--------------- 2 files changed, 101 insertions(+), 285 deletions(-) diff --git a/test/e2e/live/launch-agent-turn.ts b/test/e2e/live/launch-agent-turn.ts index da94e40e5f..6a9cd73afd 100644 --- a/test/e2e/live/launch-agent-turn.ts +++ b/test/e2e/live/launch-agent-turn.ts @@ -7,78 +7,17 @@ import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -// The live driver points only the `nemoclaw launch` process at this shim. The -// shim passes every OpenShell call through unchanged except the exact TTY exec -// that starts OpenClaw, where it attaches the non-secret run identity through -// OpenShell's supported request environment. The TUI inherits that identity, -// allowing readiness to bind to this launch instead of a process-table peer. -export const OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT = String.raw`#!/usr/bin/env node -const childProcess = require("node:child_process"); -const fs = require("node:fs"); - -const argv = process.argv.slice(2); -const realOpenShell = process.env.NEMOCLAW_LAUNCH_REAL_OPENSHELL; -const runId = process.env.NEMOCLAW_LAUNCH_RUN_ID; -const sandboxName = process.env.NEMOCLAW_LAUNCH_SANDBOX; -const interceptPath = process.env.NEMOCLAW_LAUNCH_INTERCEPT_PATH; - -function fail(reason) { - process.stderr.write(JSON.stringify({ reason }) + "\n"); - process.exit(73); -} - -function run(nextArgv) { - const result = childProcess.spawnSync(realOpenShell, nextArgv, { stdio: "inherit" }); - if (result.error || result.status === null) fail("openshell_shim_invocation_failed"); - process.exit(result.status); -} - -function arraysEqual(left, right) { - return left.length === right.length && left.every((value, index) => value === right[index]); -} - -if (!realOpenShell || !realOpenShell.startsWith("/")) fail("openshell_shim_authority_invalid"); -if (!/^[0-9a-f]{32}$/.test(runId || "")) fail("openshell_shim_run_id_invalid"); -if (!interceptPath || !interceptPath.startsWith("/")) fail("openshell_shim_path_invalid"); - -const separator = argv.indexOf("--"); -const remoteArgv = separator === -1 ? [] : argv.slice(separator + 1); -const expectedTail = ["bash", "-lc", "openclaw tui"]; -let optionIndex = 4; -if (argv[optionIndex] === "-g") optionIndex += 2; -const launchLike = - argv[0] === "sandbox" && - argv[1] === "exec" && - argv[2] === "--name" && - argv[3] === sandboxName && - arraysEqual(argv.slice(optionIndex, separator), ["--tty", "--timeout", "0"]) && - remoteArgv.length >= expectedTail.length && - expectedTail.every((value, index) => value === remoteArgv.at(index - expectedTail.length)); - -if (!launchLike) run(argv); -try { - fs.writeFileSync(interceptPath, runId + "\n", { flag: "wx", mode: 0o600 }); -} catch { - fail("openshell_launch_intercept_duplicate"); -} -run([ - ...argv.slice(0, separator), - "--env", - "NEMOCLAW_LAUNCH_RUN_ID=" + runId, - ...argv.slice(separator), -]); -`; - // OpenClaw owns the JSONL session store and does not expose a structured // result from `nemoclaw launch`. This verifier records an in-sandbox baseline, // then qualifies only complete user and assistant records appended after that // baseline. Session content never moves to the host. export const OPENCLAW_SESSION_EVIDENCE_SCRIPT = String.raw` const crypto = require("node:crypto"); +const childProcess = require("node:child_process"); const fs = require("node:fs"); const path = require("node:path"); -const [mode, sessionRoot, baselinePath, expectedTurnsText, runId] = process.argv.slice(1); +const [mode, sessionRoot, baselinePath, expectedTurnsText] = process.argv.slice(1); function finish(exitCode, reason, detail = {}) { if (reason) process.stderr.write(JSON.stringify({ reason, ...detail }) + "\n"); @@ -105,7 +44,7 @@ function sessionFileNames() { } } -function launchOwnedOpenClawTuiProcessIds() { +function openClawTuiProcessIds() { let names; try { names = fs.readdirSync("/proc"); @@ -127,27 +66,15 @@ function launchOwnedOpenClawTuiProcessIds() { } if (!args.includes("tui")) continue; if (!args.some((arg) => ["openclaw", "openclaw.mjs"].includes(path.basename(arg)))) continue; - let environment; - try { - environment = fs.readFileSync(path.join("/proc", name, "environ"), "utf8").split("\0"); - } catch (error) { - if (error && ["EACCES", "ENOENT", "ESRCH"].includes(error.code)) continue; - finish(2, "tui_environment_unavailable"); - } - if (environment.includes("NEMOCLAW_LAUNCH_RUN_ID=" + runId)) pids.push(name); + pids.push(name); } return pids; } -// The terminal line discipline safely queues a complete submitted line even -// while a canonical-mode TUI is still installing its reader. Raw mode is a UI -// implementation detail. Readiness therefore requires the exact run identity -// injected into this launch's remote exec and a PTY on that process's fd 0. -function qualifyTuiInputPty() { - if (!/^[0-9a-f]{32}$/.test(runId || "")) finish(2, "launch_run_id_invalid"); - const pids = launchOwnedOpenClawTuiProcessIds(); +function qualifyTuiInputMode() { + const pids = openClawTuiProcessIds(); if (pids.length === 0) finish(1); - if (pids.length > 1) finish(2, "multiple_launch_tui_processes"); + if (pids.length > 1) finish(2, "multiple_tui_processes"); let ttyPath; try { ttyPath = fs.realpathSync(path.join("/proc", pids[0], "fd", "0")); @@ -156,6 +83,16 @@ function qualifyTuiInputPty() { finish(2, "tui_stdin_unavailable"); } if (!/^\/dev\/pts\/\d+$/.test(ttyPath)) finish(2, "tui_stdin_not_pty"); + let state; + try { + state = childProcess.execFileSync("stty", ["-F", ttyPath, "-a"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + } catch { + finish(2, "tui_termios_unavailable"); + } + if (!/(^|[\s;])-icanon([\s;]|$)/.test(state)) finish(1); finish(0); } @@ -291,7 +228,7 @@ function qualifyTurns() { try { if (mode === "baseline") recordBaseline(); - if (mode === "input-pty") qualifyTuiInputPty(); + if (mode === "input-mode") qualifyTuiInputMode(); if (mode === "qualify") qualifyTurns(); } catch { finish(2, "verifier_failed"); @@ -308,9 +245,6 @@ capture="$session_dir/terminal.log" driver_error="$session_dir/pty-driver.err" evidence_error="$session_dir/session-evidence.err" input="$session_dir/input" -input_submitted_marker="$session_dir/input-submitted" -openshell_shim="$session_dir/openshell-launch-shim" -intercept_path="$session_dir/launch-intercept" baseline_path="/tmp/nemoclaw-launch-session-$NEMOCLAW_LAUNCH_RUN_ID.json" session_pid="" session_deadline="" @@ -389,8 +323,7 @@ session_evidence() { "$mode" \ "$NEMOCLAW_LAUNCH_SESSION_ROOT" \ "$baseline_path" \ - "$expected_turns" \ - "$NEMOCLAW_LAUNCH_RUN_ID" + "$expected_turns" } wait_for_turn_count() { @@ -413,31 +346,29 @@ wait_for_turn_count() { fail_launch_session "launch did not record the required structured session turns" } -wait_for_tui_input_pty() { +wait_for_pty_input_mode() { local evidence_status while (( SECONDS < session_deadline )); do - if session_evidence input-pty >/dev/null 2>"$evidence_error"; then + if session_evidence input-mode >/dev/null 2>"$evidence_error"; then return 0 else evidence_status=$? fi if [[ "$evidence_status" != 1 ]]; then - fail_launch_session "OpenClaw TUI input PTY evidence was invalid or unavailable (status $evidence_status)" + fail_launch_session "OpenClaw TUI input-mode evidence was invalid or unavailable (status $evidence_status)" fi if ! kill -0 "$session_pid" 2>/dev/null; then break fi sleep 0.1 done - fail_launch_session "OpenClaw TUI did not attach standard input to the launch PTY before the session deadline" + fail_launch_session "launch PTY did not enter input mode before the session deadline" } if ! session_evidence baseline >/dev/null 2>"$evidence_error"; then fail_launch_session "launch could not record the structured session baseline" fi -printf '%s' "$NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT" >"$openshell_shim" -chmod 700 "$openshell_shim" mkfifo -m 600 "$input" if [[ -n "$NEMOCLAW_LAUNCH_ENTRYPOINT" ]]; then printf -v launch_command '%q %q %q %q' \ @@ -448,10 +379,6 @@ else "$NEMOCLAW_LAUNCH_COMMAND" launch "$NEMOCLAW_LAUNCH_SANDBOX" fi -NEMOCLAW_LAUNCH_INPUT_SUBMITTED_MARKER="$input_submitted_marker" \ -NEMOCLAW_LAUNCH_INTERCEPT_PATH="$intercept_path" \ -NEMOCLAW_LAUNCH_REAL_OPENSHELL="$NEMOCLAW_OPENSHELL_COMMAND" \ -NEMOCLAW_OPENSHELL_BIN="$openshell_shim" \ timeout --kill-after=5s 250s \ script --quiet --return --flush --command "$launch_command" "$capture" \ <"$input" >/dev/null 2>"$driver_error" & @@ -475,11 +402,10 @@ if [[ "$capture_ready" != 1 ]]; then fail_launch_session "launch did not create a PTY diagnostic capture" fi -wait_for_tui_input_pty +wait_for_pty_input_mode if ! printf '%s\r' "$NEMOCLAW_LAUNCH_FIRST_INPUT" >&3; then fail_launch_session "launch exited before the first PTY input was submitted" fi -: >"$input_submitted_marker" wait_for_turn_count 1 if ! printf '%s\r' "$NEMOCLAW_LAUNCH_SECOND_INPUT" >&3; then fail_launch_session "launch exited before the second PTY input was submitted" @@ -548,9 +474,6 @@ export async function runOpenClawLaunchSession( if (process.platform !== "linux") { throw new Error("launch session coverage requires the Linux util-linux PTY driver"); } - if (!options.host.openshellCommandPath.startsWith("/")) { - throw new Error("launch session coverage requires an absolute OpenShell command path"); - } const inputs = uniqueTurnInputs(); const result = await options.host.command("bash", ["-lc", LAUNCH_TURN_SCRIPT], { artifactName: options.artifactName, @@ -560,7 +483,6 @@ export async function runOpenClawLaunchSession( NEMOCLAW_LAUNCH_ENTRYPOINT: options.cliEntrypoint ?? "", NEMOCLAW_LAUNCH_EXIT_COMMAND: options.exitCommand ?? "", NEMOCLAW_LAUNCH_FIRST_INPUT: inputs.first, - NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT: OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, NEMOCLAW_LAUNCH_RUN_ID: randomUUID().replaceAll("-", ""), NEMOCLAW_LAUNCH_SANDBOX: options.sandboxName, NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: "230", diff --git a/test/e2e/support/launch-agent-turn.test.ts b/test/e2e/support/launch-agent-turn.test.ts index 3496a7a6d8..8c319e75b7 100644 --- a/test/e2e/support/launch-agent-turn.test.ts +++ b/test/e2e/support/launch-agent-turn.test.ts @@ -1,8 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn, spawnSync } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { spawnSync } from "node:child_process"; import { appendFileSync, chmodSync, @@ -15,12 +14,11 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { expect, it } from "vitest"; import { LAUNCH_TURN_SCRIPT, - OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, OPENCLAW_SESSION_EVIDENCE_SCRIPT, runOpenClawLaunchReadinessLeaseTurns, } from "../live/launch-agent-turn.ts"; @@ -30,11 +28,11 @@ const PROCESS_EXIT_WAIT = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_ type SessionRecords = Record; type FixtureMode = | "cleanup-failure" - | "delayed-input-reader" + | "delayed-input-attachment" | "delayed-recording" + | "input-mode-timeout" | "invalid-order" | "late-extra" - | "mismatched-tui-pty" | "multiple-tui-processes" | "nonzero" | "nonzero-cleanup-failure" @@ -148,46 +146,11 @@ function runLaunchSessionFixture(mode: FixtureMode, terminalCopy: "absent" | "an const tuiStdinRetryMarker = join(fixtureRoot, "tui-stdin-retry"); const tuiStdinUnavailableMarker = join(fixtureRoot, "tui-stdin-unavailable"); const ttyMarker = join(fixtureRoot, "tty-observed"); - const launchTtyMarker = join(fixtureRoot, "launch-tty"); - const unrelatedTtyMarker = join(fixtureRoot, "unrelated-tty"); - const readerOrderMarker = join(fixtureRoot, "reader-after-submission"); - const runId = randomUUID().replaceAll("-", ""); + const runId = basename(fixtureRoot).replaceAll(/[^a-zA-Z0-9]/gu, ""); const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; - let unrelatedScriptProcess: ReturnType | null = null; mkdirSync(sessionRoot); mkdirSync(tuiInputMarkerRoot); - const recordedTuiProcessIds = (): string[] => - existsSync(tuiPidsPath) - ? readFileSync(tuiPidsPath, "utf8").trim().split("\n").filter(Boolean) - : []; - const processIsRunning = (pid: string): boolean => { - try { - const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); - return stat.slice(stat.lastIndexOf(")") + 2, stat.lastIndexOf(")") + 3) !== "Z"; - } catch { - return false; - } - }; - const stopUnrelatedTui = (): void => { - const stopAndWait = (processIds: string[]): void => { - for (const signal of ["SIGTERM", "SIGKILL"] as const) { - for (const pid of processIds) { - try { - process.kill(Number(pid), signal); - } catch {} - } - const exitDeadline = Date.now() + 1_000; - while (processIds.some(processIsRunning) && Date.now() < exitDeadline) { - Atomics.wait(PROCESS_EXIT_WAIT, 0, 0, 25); - } - } - }; - stopAndWait(recordedTuiProcessIds()); - stopAndWait(unrelatedScriptProcess?.pid ? [String(unrelatedScriptProcess.pid)] : []); - unrelatedScriptProcess = null; - }; - try { writeFileSync( fakeLaunch, @@ -198,14 +161,7 @@ const childProcess = require("node:child_process"); const mode = process.env.NEMOCLAW_FIXTURE_MODE; if (process.argv[2] !== "tui") { - if (mode === "mismatched-tui-pty") { - fs.writeFileSync( - process.env.NEMOCLAW_FIXTURE_LAUNCH_TTY_MARKER, - fs.realpathSync("/proc/self/fd/0"), - ); - process.stdin.resume(); - process.stdin.once("end", () => process.exit(0)); - } else if (mode === "transient-tui-stdin") { + if (mode === "transient-tui-stdin") { const transient = childProcess.spawn( process.execPath, [__filename, "tui", "stdin-unavailable"], @@ -270,13 +226,6 @@ if (process.argv[2] !== "tui") { if (process.argv[2] === "tui") (async () => { if (!process.stdin.isTTY || !process.stdout.isTTY) process.exit(64); fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_TTY_MARKER, ""); - if (process.argv[3] === "unrelated") { - fs.appendFileSync(process.env.NEMOCLAW_FIXTURE_TUI_PIDS, process.pid + "\n"); - fs.writeFileSync( - process.env.NEMOCLAW_FIXTURE_UNRELATED_TTY_MARKER, - fs.realpathSync("/proc/self/fd/0"), - ); - } const sessionFile = process.env.NEMOCLAW_FIXTURE_SESSION_FILE; const terminalCopy = process.env.NEMOCLAW_FIXTURE_TERMINAL_COPY; const append = (role, content) => fs.appendFileSync( @@ -284,7 +233,6 @@ if (process.argv[2] === "tui") (async () => { JSON.stringify({ message: { content: [{ text: content, type: "text" }], role }, type: "message" }) + "\n", ); if ( - mode === "mismatched-tui-pty" || mode === "multiple-tui-processes" || (mode === "transient-tui-stdin" && process.argv[3] !== "stdin-unavailable") ) { @@ -308,11 +256,13 @@ if (process.argv[2] === "tui") (async () => { } process.exit(fs.existsSync(process.env.NEMOCLAW_FIXTURE_TUI_STDIN_RETRY) ? 0 : 68); } - if (mode === "delayed-input-reader") { - while (!fs.existsSync(process.env.NEMOCLAW_LAUNCH_INPUT_SUBMITTED_MARKER)) { - await new Promise((resolve) => setTimeout(resolve, 10)); - } - fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_READER_ORDER_MARKER, ""); + if (mode === "delayed-input-attachment" || mode === "input-mode-timeout") { + let inputBeforeAttachment = false; + const recordEarlyInput = () => { inputBeforeAttachment = true; }; + process.stdin.on("data", recordEarlyInput); + await new Promise((resolve) => setTimeout(resolve, mode === "input-mode-timeout" ? 10_000 : 1_500)); + process.stdin.off("data", recordEarlyInput); + if (inputBeforeAttachment) process.exit(67); } const rl = readline.createInterface({ input: process.stdin, output: process.stdout, terminal: true }); const ask = () => new Promise((resolve) => rl.question("", resolve)); @@ -361,18 +311,10 @@ set -euo pipefail if [[ "$NEMOCLAW_FIXTURE_MODE" == *"cleanup-failure" && " $* " == *" rm -f -- "* ]]; then exit 71 fi -injected_env="" -while [[ "$#" -gt 0 && "$1" != "--" ]]; do - if [[ "$1" == "--env" ]]; then - injected_env="$2" - shift 2 - else - shift - fi -done +while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done [[ "$#" -gt 0 ]] shift -if [[ "$NEMOCLAW_FIXTURE_MODE" == "transient-tui-stdin" && "$4" == "input-pty" ]]; then +if [[ "$NEMOCLAW_FIXTURE_MODE" == "transient-tui-stdin" && "$4" == "input-mode" ]]; then set +e "$@" status=$? @@ -382,104 +324,63 @@ if [[ "$NEMOCLAW_FIXTURE_MODE" == "transient-tui-stdin" && "$4" == "input-pty" ] fi exit "$status" fi -if [[ -n "$injected_env" ]]; then - exec env -u NEMOCLAW_LAUNCH_RUN_ID "$injected_env" "$@" -fi -exec env -u NEMOCLAW_LAUNCH_RUN_ID "$@" +exec "$@" `, ); chmodSync(fakeLaunch, 0o755); chmodSync(fakeOpenshell, 0o755); - const fixtureEnv: NodeJS.ProcessEnv = { - ...process.env, - NEMOCLAW_FIXTURE_MODE: mode, - NEMOCLAW_FIXTURE_LAUNCH_TTY_MARKER: launchTtyMarker, - NEMOCLAW_FIXTURE_NODE: process.execPath, - NEMOCLAW_FIXTURE_READER_ORDER_MARKER: readerOrderMarker, - NEMOCLAW_FIXTURE_SCRIPT: fakeLaunch, - NEMOCLAW_FIXTURE_SESSION_FILE: join(sessionRoot, "session-a.jsonl"), - NEMOCLAW_FIXTURE_TERMINAL_COPY: terminalCopy, - NEMOCLAW_FIXTURE_TUI_INPUT_MARKER_ROOT: tuiInputMarkerRoot, - NEMOCLAW_FIXTURE_TUI_PIDS: tuiPidsPath, - NEMOCLAW_FIXTURE_TUI_STDIN_RETRY: tuiStdinRetryMarker, - NEMOCLAW_FIXTURE_TUI_STDIN_UNAVAILABLE: tuiStdinUnavailableMarker, - NEMOCLAW_FIXTURE_TTY_MARKER: ttyMarker, - NEMOCLAW_FIXTURE_UNRELATED_TTY_MARKER: unrelatedTtyMarker, - NEMOCLAW_LAUNCH_COMMAND: fakeLaunch, - NEMOCLAW_LAUNCH_ENTRYPOINT: "", - NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", - NEMOCLAW_LAUNCH_FIRST_INPUT: "first input", - NEMOCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT: OPENCLAW_LAUNCH_OPENSHELL_SHIM_SCRIPT, - NEMOCLAW_LAUNCH_RUN_ID: runId, - NEMOCLAW_LAUNCH_SANDBOX: "sandbox", - NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: - mode.endsWith("-timeout") || mode === "mismatched-tui-pty" ? "2" : "230", - NEMOCLAW_LAUNCH_SECOND_INPUT: "second input", - NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, - NEMOCLAW_LAUNCH_SESSION_ROOT: sessionRoot, - NEMOCLAW_OPENSHELL_COMMAND: fakeOpenshell, - TERM: "xterm-256color", - }; - const startUnrelatedTui = (): void => { - unrelatedScriptProcess = spawn( - "script", - [ - "--quiet", - "--return", - "--command", - '"$NEMOCLAW_FIXTURE_NODE" "$NEMOCLAW_FIXTURE_SCRIPT" tui unrelated', - "/dev/null", - ], - { - env: { - ...fixtureEnv, - NEMOCLAW_LAUNCH_RUN_ID: "00000000000000000000000000000000", - }, - stdio: "ignore", - }, - ); - const unrelatedReadyDeadline = Date.now() + 5_000; - while ( - !existsSync(unrelatedTtyMarker) && - unrelatedScriptProcess.pid && - existsSync(`/proc/${unrelatedScriptProcess.pid}`) && - Date.now() < unrelatedReadyDeadline - ) { - Atomics.wait(PROCESS_EXIT_WAIT, 0, 0, 25); - } - expect(existsSync(unrelatedTtyMarker), "unrelated TUI fixture PTY identity").toBe(true); - }; - const setupFixture = mode === "mismatched-tui-pty" ? startUnrelatedTui : () => undefined; - setupFixture(); - const result = spawnSync("bash", ["-c", LAUNCH_TURN_SCRIPT], { encoding: "utf8", killSignal: "SIGKILL", - env: fixtureEnv, + env: { + ...process.env, + NEMOCLAW_FIXTURE_MODE: mode, + NEMOCLAW_FIXTURE_SESSION_FILE: join(sessionRoot, "session-a.jsonl"), + NEMOCLAW_FIXTURE_TERMINAL_COPY: terminalCopy, + NEMOCLAW_FIXTURE_TUI_INPUT_MARKER_ROOT: tuiInputMarkerRoot, + NEMOCLAW_FIXTURE_TUI_PIDS: tuiPidsPath, + NEMOCLAW_FIXTURE_TUI_STDIN_RETRY: tuiStdinRetryMarker, + NEMOCLAW_FIXTURE_TUI_STDIN_UNAVAILABLE: tuiStdinUnavailableMarker, + NEMOCLAW_FIXTURE_TTY_MARKER: ttyMarker, + NEMOCLAW_LAUNCH_COMMAND: fakeLaunch, + NEMOCLAW_LAUNCH_ENTRYPOINT: "", + NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", + NEMOCLAW_LAUNCH_FIRST_INPUT: "first input", + NEMOCLAW_LAUNCH_RUN_ID: runId, + NEMOCLAW_LAUNCH_SANDBOX: "sandbox", + NEMOCLAW_LAUNCH_SESSION_BUDGET_SECONDS: mode.endsWith("-timeout") ? "2" : "230", + NEMOCLAW_LAUNCH_SECOND_INPUT: "second input", + NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, + NEMOCLAW_LAUNCH_SESSION_ROOT: sessionRoot, + NEMOCLAW_OPENSHELL_COMMAND: fakeOpenshell, + TERM: "xterm-256color", + }, timeout: 15_000, }); - const tuiProcessIds = recordedTuiProcessIds(); - stopUnrelatedTui(); + const tuiProcessIds = existsSync(tuiPidsPath) + ? readFileSync(tuiPidsPath, "utf8").trim().split("\n").filter(Boolean) + : []; + const processExitDeadline = Date.now() + 1_000; + while ( + tuiProcessIds.some((pid) => existsSync(`/proc/${pid}`)) && + Date.now() < processExitDeadline + ) { + Atomics.wait(PROCESS_EXIT_WAIT, 0, 0, 25); + } return { baselineRemoved: !existsSync(baselinePath), - orphanedTuiProcessIds: tuiProcessIds.filter(processIsRunning), + orphanedTuiProcessIds: tuiProcessIds.filter((pid) => existsSync(`/proc/${pid}`)), recordedTuiInputProcessIds: tuiProcessIds.filter((pid) => existsSync(join(tuiInputMarkerRoot, pid)), ), result, - launchTty: existsSync(launchTtyMarker) ? readFileSync(launchTtyMarker, "utf8") : null, - readerInstalledAfterSubmission: existsSync(readerOrderMarker), tuiStdinUnavailableObserved: existsSync(tuiStdinUnavailableMarker), tuiProcessIds, ttyObserved: existsSync(ttyMarker), - unrelatedTty: existsSync(unrelatedTtyMarker) - ? readFileSync(unrelatedTtyMarker, "utf8") - : null, }; } finally { - stopUnrelatedTui(); rmSync(fixtureRoot, { force: true, recursive: true }); rmSync(baselinePath, { force: true }); } @@ -586,45 +487,20 @@ it.runIf(process.platform === "linux")( ); it.runIf(process.platform === "linux")( - "queues one canonical PTY line until the OpenClaw TUI installs its input reader (#9160)", + "waits for the OpenClaw TUI input mode before submitting PTY input (#9160)", () => { - const { baselineRemoved, readerInstalledAfterSubmission, result, ttyObserved } = - runLaunchSessionFixture("delayed-input-reader", "absent"); + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "delayed-input-attachment", + "absent", + ); expect(ttyObserved).toBe(true); - expect(readerInstalledAfterSubmission).toBe(true); expect(baselineRemoved).toBe(true); expect(result.signal).toBeNull(); expect(result.status).toBe(0); }, ); -it.runIf(process.platform === "linux")( - "rejects an unrelated OpenClaw TUI on a different PTY before submitting input (#9160)", - () => { - const { - baselineRemoved, - launchTty, - orphanedTuiProcessIds, - recordedTuiInputProcessIds, - result, - unrelatedTty, - } = runLaunchSessionFixture("mismatched-tui-pty", "absent"); - - expect(launchTty).toMatch(/^\/dev\/pts\/\d+$/); - expect(unrelatedTty).toMatch(/^\/dev\/pts\/\d+$/); - expect(unrelatedTty).not.toBe(launchTty); - expect(recordedTuiInputProcessIds).toEqual([]); - expect(orphanedTuiProcessIds).toEqual([]); - expect(baselineRemoved).toBe(true); - expect(result.signal).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "OpenClaw TUI did not attach standard input to the launch PTY before the session deadline", - ); - }, -); - it.runIf(process.platform === "linux")( "retries when a matching OpenClaw TUI process closes standard input (#9160)", () => { @@ -668,7 +544,7 @@ it.runIf(process.platform === "linux")( expect(baselineRemoved).toBe(true); expect(result.signal).toBeNull(); expect(result.status).toBe(1); - expect(result.stderr).toContain('"reason":"multiple_launch_tui_processes"'); + expect(result.stderr).toContain('"reason":"multiple_tui_processes"'); }, ); @@ -687,6 +563,24 @@ it.runIf(process.platform === "linux")( }, ); +it.runIf(process.platform === "linux")( + "reports a missing OpenClaw input mode before the PTY child timeout (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "input-mode-timeout", + "absent", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "launch PTY did not enter input mode before the session deadline", + ); + }, +); + it.runIf(process.platform === "linux")( "reports missing structured turns before the PTY child timeout (#9160)", () => { @@ -789,7 +683,7 @@ it.runIf(process.platform === "linux")( calls.push({ command, args, env: options?.env }); return { exitCode: 0, signal: null, stdout: "", stderr: "" }; }, - openshellCommandPath: "/usr/bin/openshell", + openshellCommandPath: "openshell", }; await runOpenClawLaunchReadinessLeaseTurns({ @@ -822,8 +716,8 @@ it.runIf(process.platform === "linux")( "/exit", ]); expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_OPENSHELL_COMMAND)).toEqual([ - "/usr/bin/openshell", - "/usr/bin/openshell", + "openshell", + "openshell", ]); for (const call of calls.slice(1)) { expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_EXPECTED_REPLY"); From dfba140492533c2c3c6438b8a82c32a0514a1620 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 16 Aug 2026 08:20:49 -0700 Subject: [PATCH 8/8] docs: clarify direct route validation recovery Signed-off-by: Prekshi Vyas --- docs/inference/switch-providers.mdx | 6 +++--- docs/reference/commands.mdx | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/inference/switch-providers.mdx b/docs/inference/switch-providers.mdx index d0f793761e..2f894e5199 100644 --- a/docs/inference/switch-providers.mdx +++ b/docs/inference/switch-providers.mdx @@ -73,10 +73,10 @@ Run the rebuild before relying on the running agent. Use `--no-verify` only when OpenShell cannot verify the target provider at switch time and you have already confirmed its provider and credential. This flag does not bypass shared-gateway compatibility checks. -When you explicitly supply a direct compatible-provider endpoint at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. -It then sends one minimal request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens. +When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +It then sends one validation request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens. If that request fails, NemoClaw attempts to restore the previous OpenShell selection and remove a provider that this switch created. -If the error reports that rollback could not complete, re-run onboarding before using or retrying the route. +If the error reports that rollback could not complete, rerun onboarding before using the route or retrying the switch. Endpoint-shape and shared-gateway compatibility checks still apply. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index e7954fedf5..04c5523234 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -4100,10 +4100,10 @@ If the in-sandbox config sync fails, NemoClaw keeps the gateway and registry ali Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `hermes-provider`, `ollama-local`, and `vllm-local`. Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential. -When you explicitly supply a direct compatible-provider endpoint at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. -Before it persists the route in the NemoClaw registry or agent config, the command sends one request from the target sandbox with a 16-token output limit. +When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +Before it persists the route in the NemoClaw registry or agent config, the command sends one validation request from the target sandbox with a 16-token output limit. If that request fails, the command attempts to restore the previous OpenShell selection and remove a provider that this switch created. -If the error reports that rollback could not complete, re-run onboarding before using or retrying the route. +If the error reports that rollback could not complete, rerun onboarding before using the route or retrying the switch. Endpoint-shape and shared-gateway compatibility checks still apply. When switching to `compatible-endpoint` or `compatible-anthropic-endpoint` from a different provider family, pass `--endpoint-url` with the trusted custom provider URL and, except for the Hermes case below, `--inference-api` with its API family so NemoClaw can persist a complete route identity for rebuild and shared-gateway checks. For a Hermes `compatible-anthropic-endpoint` target, `--inference-api` may be omitted because NemoClaw deterministically selects `openai-completions`; an explicit different API family is rejected.