diff --git a/docs/inference/switch-providers.mdx b/docs/inference/switch-providers.mdx index 9b84f2148c..2f894e5199 100644 --- a/docs/inference/switch-providers.mdx +++ b/docs/inference/switch-providers.mdx @@ -73,6 +73,11 @@ Run the rebuild before relying on the running agent. Use `--no-verify` only when OpenShell cannot verify the target provider at switch time and you have already confirmed its provider and credential. This flag does not bypass shared-gateway compatibility checks. +When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +It then sends one validation request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens. +If that request fails, NemoClaw attempts to restore the previous OpenShell selection and remove a provider that this switch created. +If the error reports that rollback could not complete, rerun onboarding before using the route or retrying the switch. +Endpoint-shape and shared-gateway compatibility checks still apply. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index aa56e91fc1..04c5523234 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -4100,6 +4100,11 @@ If the in-sandbox config sync fails, NemoClaw keeps the gateway and registry ali Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `hermes-provider`, `ollama-local`, and `vllm-local`. Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential. +When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network. +Before it persists the route in the NemoClaw registry or agent config, the command sends one validation request from the target sandbox with a 16-token output limit. +If that request fails, the command attempts to restore the previous OpenShell selection and remove a provider that this switch created. +If the error reports that rollback could not complete, rerun onboarding before using the route or retrying the switch. +Endpoint-shape and shared-gateway compatibility checks still apply. When switching to `compatible-endpoint` or `compatible-anthropic-endpoint` from a different provider family, pass `--endpoint-url` with the trusted custom provider URL and, except for the Hermes case below, `--inference-api` with its API family so NemoClaw can persist a complete route identity for rebuild and shared-gateway checks. For a Hermes `compatible-anthropic-endpoint` target, `--inference-api` may be omitted because NemoClaw deterministically selects `openai-completions`; an explicit different API family is rejected. NemoClaw rejects loopback, link-local, private, and internal endpoint addresses, including public hostnames that resolve to a private address. diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index 803e6d6487..85b343c05f 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -616,7 +616,6 @@ describe("runInferenceSet compatible providers", () => { { provider: "compatible-anthropic-endpoint", model: "mock-anthropic-model", - noVerify: true, endpointUrl: "http://host.openshell.internal:18767/", credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", inferenceApi: "anthropic-messages", @@ -644,6 +643,221 @@ describe("runInferenceSet compatible providers", () => { nimContainer: null, }); expect(deps.calls.rewriteConfigUrlsWithDnsPinning).not.toHaveBeenCalled(); + expect(captureOpenshell).toHaveBeenCalledWith( + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "compatible-anthropic-endpoint", + "--model", + "mock-anthropic-model", + "--no-verify", + ], + expect.objectContaining({ ignoreError: true }), + ); + expect(deps.calls.probeSandboxRoute).toHaveBeenCalledWith({ + sandboxName: "alpha", + provider: "compatible-anthropic-endpoint", + model: "mock-anthropic-model", + preferredInferenceApi: "anthropic-messages", + }); + expect(deps.calls.probeSandboxRoute.mock.invocationCallOrder[0]).toBeLessThan( + deps.calls.updateSandbox.mock.invocationCallOrder[0], + ); + }); + + it.each([ + [ + "returns a rejection", + () => ({ + ok: false, + detail: "sandbox inference invocation probe exited with status 7", + httpStatus: null, + }), + /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s, + ], + [ + "throws", + () => { + throw new Error("sandbox dial failed"); + }, + /sandbox inference invocation probe was unavailable: sandbox dial failed.*previous OpenShell inference selection was restored/s, + ], + ])("restores the prior route when sandbox-only provider verification %s", async (_failureMode, probeSandboxRoute, expectedError) => { + const captureOpenshell = createCompatibleProviderCapture({ + name: "compatible-anthropic-endpoint", + type: "anthropic", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + configKey: "ANTHROPIC_BASE_URL", + initiallyPresent: false, + }); + const deps = createDeps({ + config: { agents: { defaults: { model: { primary: "inference/old-model" } } } }, + entry: { + name: "alpha", + agent: "openclaw", + provider: "nvidia-prod", + model: "old-model", + }, + session: baseSession({ provider: "nvidia-prod", model: "old-model" }), + captureOpenshell, + probeSandboxRoute, + }); + + await expect( + runInferenceSet( + { + provider: "compatible-anthropic-endpoint", + model: "mock-anthropic-model", + endpointUrl: "http://host.openshell.internal:18767/", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + inferenceApi: "anthropic-messages", + }, + deps, + ), + ).rejects.toThrow(expectedError); + + expect( + captureOpenshell.mock.calls + .filter(([args]) => args[0] === "inference" && args[1] === "set") + .map(([args]) => args), + ).toEqual([ + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "compatible-anthropic-endpoint", + "--model", + "mock-anthropic-model", + "--no-verify", + ], + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "nvidia-prod", + "--model", + "old-model", + "--no-verify", + ], + ]); + expect( + captureOpenshell.mock.calls.some( + ([args]) => args[0] === "provider" && args[1] === "delete", + ), + ).toBe(true); + expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); + expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); + }); + + it("preserves redacted probe diagnostics when restoring the prior route fails", async () => { + const providerCapture = createCompatibleProviderCapture({ + name: "compatible-anthropic-endpoint", + type: "anthropic", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + configKey: "ANTHROPIC_BASE_URL", + initiallyPresent: false, + }); + const inferenceSetResults = [ + null, + { + status: 19, + output: "restore rejected", + stdout: "", + stderr: "restore rejected", + }, + ]; + let inferenceSetCalls = 0; + const captureOpenshell = vi.fn((args: string[]) => { + switch (`${args[0]}:${args[1]}`) { + case "inference:set": + return inferenceSetResults[inferenceSetCalls++] ?? providerCapture(args); + default: + return providerCapture(args); + } + }); + const deps = createDeps({ + config: { agents: { defaults: { model: { primary: "inference/old-model" } } } }, + entry: { + name: "alpha", + agent: "openclaw", + provider: "nvidia-prod", + model: "old-model", + }, + session: baseSession({ provider: "nvidia-prod", model: "old-model" }), + captureOpenshell, + probeSandboxRoute: () => { + throw new Error("sandbox dial failed; NVIDIA_API_KEY=nvapi-secret-value"); + }, + }); + + let failure: unknown; + try { + await runInferenceSet( + { + provider: "compatible-anthropic-endpoint", + model: "mock-anthropic-model", + endpointUrl: "http://host.openshell.internal:18767/", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + inferenceApi: "anthropic-messages", + }, + deps, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(Error); + const failureMessage = (failure as Error).message; + expect(failureMessage).toContain( + "sandbox inference invocation probe was unavailable: sandbox dial failed", + ); + expect(failureMessage).toContain("NVIDIA_API_KEY="); + expect(failureMessage).not.toContain("nvapi-secret-value"); + expect(failureMessage).toMatch( + /Failed to restore the previous OpenShell inference selection.*status 19.*Re-run onboarding/s, + ); + expect( + deps.calls.captureOpenshell.mock.calls + .filter(([args]) => args[0] === "inference" && args[1] === "set") + .map(([args]) => args), + ).toEqual([ + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "compatible-anthropic-endpoint", + "--model", + "mock-anthropic-model", + "--no-verify", + ], + [ + "inference", + "set", + "-g", + "nemoclaw", + "--provider", + "nvidia-prod", + "--model", + "old-model", + "--no-verify", + ], + ]); + expect( + deps.calls.captureOpenshell.mock.calls.some( + ([args]) => args[0] === "provider" && args[1] === "delete", + ), + ).toBe(false); + expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); + expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); }); for (const provider of ["compatible-endpoint", "compatible-anthropic-endpoint"]) { diff --git a/src/lib/actions/inference-set-provider.ts b/src/lib/actions/inference-set-provider.ts index 8bcf01545c..daf4cef5cc 100644 --- a/src/lib/actions/inference-set-provider.ts +++ b/src/lib/actions/inference-set-provider.ts @@ -20,10 +20,31 @@ import { openshellReportsProviderNotFound, } from "./inference-set-error"; import type { InferenceSetProviderBinding } from "./inference-set-route-containment"; +import type { + SandboxInferenceInvocationInput, + SandboxInferenceInvocationResult, +} from "./sandbox/inference-invocation-probe"; export type { RuntimeProviderBundleRegistry }; export { RuntimeProviderSelectionError }; +export type InferenceSetSandboxRouteProbe = ( + input: SandboxInferenceInvocationInput, +) => SandboxInferenceInvocationResult; + +export function probeInferenceSetSandboxRoute( + input: SandboxInferenceInvocationInput, +): SandboxInferenceInvocationResult { + const probe: typeof import("./sandbox/inference-invocation-probe") = require( + "./sandbox/inference-invocation-probe", + ); + return probe.probeSandboxInferenceInvocation( + input, + {}, + probe.READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, + ); +} + export function requireInferenceSetRuntimeAuthority( entry: SandboxEntry, providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, diff --git a/src/lib/actions/inference-set-route-containment.ts b/src/lib/actions/inference-set-route-containment.ts index 4625e3559e..fb6d250ad7 100644 --- a/src/lib/actions/inference-set-route-containment.ts +++ b/src/lib/actions/inference-set-route-containment.ts @@ -73,6 +73,13 @@ export interface HttpsPinProviderBinding extends InferenceSetProviderBinding { routeId: string; } +/** OpenShell's host verifier cannot resolve routes exposed only on its sandbox bridge. */ +export function isSandboxBridgeProviderBinding( + binding: InferenceSetProviderBinding | null, +): boolean { + return binding !== null && isAllowedOpenShellSandboxBridgeUrl(new URL(binding.baseUrl)); +} + type EnsureHttpsPinAdapterRoute = (endpointUrl: string) => Promise; export interface PreparedInferenceSetRoute { diff --git a/src/lib/actions/inference-set.test-support.ts b/src/lib/actions/inference-set.test-support.ts index 45336d2029..94cb2b679e 100644 --- a/src/lib/actions/inference-set.test-support.ts +++ b/src/lib/actions/inference-set.test-support.ts @@ -135,6 +135,7 @@ export function createDeps(options: { resolveCredentialValue?: InferenceSetDeps["resolveCredentialValue"]; ensureHttpsPinRuntimeAdapter?: EnsureHttpsPinRuntimeAdapterFn; revokeHttpsPinRuntimeAdapterRoute?: InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"]; + probeSandboxRoute?: InferenceSetDeps["probeSandboxRoute"]; updateSandbox?: InferenceSetDeps["updateSandbox"]; restartSandboxGateway?: InferenceSetDeps["restartSandboxGateway"]; seedHermesDashboardConfigResult?: "converged" | "absent" | "failed"; @@ -158,6 +159,7 @@ export function createDeps(options: { resolveCredentialValue: ReturnType; ensureHttpsPinRuntimeAdapter: ReturnType; revokeHttpsPinRuntimeAdapterRoute: ReturnType; + probeSandboxRoute: ReturnType; restartSandboxGateway: ReturnType; withGatewayRouteMutationLock: ReturnType; }; @@ -218,6 +220,7 @@ export function createDeps(options: { revokeHttpsPinRuntimeAdapterRoute: vi.fn( options.revokeHttpsPinRuntimeAdapterRoute ?? (async () => true), ), + probeSandboxRoute: vi.fn(options.probeSandboxRoute ?? (() => ({ ok: true }) as const)), restartSandboxGateway: vi.fn( options.restartSandboxGateway ?? ((): ReturnType => ({ @@ -262,6 +265,7 @@ export function createDeps(options: { calls.ensureHttpsPinRuntimeAdapter as unknown as EnsureHttpsPinRuntimeAdapterFn, revokeHttpsPinRuntimeAdapterRoute: calls.revokeHttpsPinRuntimeAdapterRoute as InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"], + probeSandboxRoute: calls.probeSandboxRoute as InferenceSetDeps["probeSandboxRoute"], withGatewayRouteMutationLock: calls.withGatewayRouteMutationLock as InferenceSetDeps["withGatewayRouteMutationLock"], restartSandboxGateway: calls.restartSandboxGateway, diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index d0e3482524..c6fcf0d0f1 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -71,7 +71,9 @@ import { readPreviousOpenClawInferenceApi, } from "./inference-set-gateway-restart"; import { + type InferenceSetSandboxRouteProbe, prepareInferenceSetProviderBinding, + probeInferenceSetSandboxRoute, type RuntimeProviderBundleRegistry, RuntimeProviderSelectionError, requireInferenceSetRuntimeAuthority, @@ -85,6 +87,7 @@ import { type EnsureHttpsPinRuntimeAdapterFn, finalizeInferenceSetRoute, type InferenceSetProviderBinding, + isSandboxBridgeProviderBinding, prepareInferenceSetRoute, type RegistryInferenceMetadata, } from "./inference-set-route-containment"; @@ -163,6 +166,7 @@ export interface InferenceSetDeps extends InferenceGatewayRestartDeps { resolveCredentialValue: (credentialEnv: string) => string; ensureHttpsPinRuntimeAdapter: EnsureHttpsPinRuntimeAdapterFn; revokeHttpsPinRuntimeAdapterRoute: (routeId: string) => Promise; + probeSandboxRoute: InferenceSetSandboxRouteProbe; withGatewayRouteMutationLock: typeof withGatewayRouteMutationLock; } @@ -268,6 +272,7 @@ function defaultDeps(): InferenceSetDeps { resolveCredentialValue: (credentialEnv) => process.env[credentialEnv] ?? "", ensureHttpsPinRuntimeAdapter, revokeHttpsPinRuntimeAdapterRoute, + probeSandboxRoute: probeInferenceSetSandboxRoute, withGatewayRouteMutationLock, restartSandboxGateway: defaultInferenceGatewayRestart, isSandboxConfigMutable: (sandboxName) => { @@ -936,10 +941,15 @@ async function runInferenceSetWithoutHostLock( // verify. Only a genuinely-unreachable host stack hard-fails here, before the // route is touched. let effectiveNoVerify = options.noVerify === true; - // The adapter origin resolves only from inside the sandbox network. The - // host-side OpenShell verifier cannot resolve host.openshell.internal, so - // adapter registration + local health are the verification boundary. - if (httpsPinProviderBinding) effectiveNoVerify = true; + const probeDirectSandboxBridge = isSandboxBridgeProviderBinding(directProviderBinding); + // Adapter routes and explicit custom routes on NemoClaw's sandbox bridge + // resolve only from inside the sandbox network. The host-side OpenShell + // verifier cannot resolve host.openshell.internal, so its result would be a + // guaranteed false negative. HTTPS-pin adapters retain their local-health + // verification; direct bridge routes are probed from the sandbox below. + if (httpsPinProviderBinding || probeDirectSandboxBridge) { + effectiveNoVerify = true; + } if (deps.isLocalInferenceProvider(provider)) { const localValidation = deps.validateLocalProvider(provider); if (localValidation.ok) { @@ -996,11 +1006,43 @@ async function runInferenceSetWithoutHostLock( assertReasoningEffortRoute(reasoningEffortRequest, provider, preMutationInferenceApi); const previousProvider = typeof entry.provider === "string" ? entry.provider.trim() : ""; const previousModel = typeof entry.model === "string" ? entry.model.trim() : ""; + if (probeDirectSandboxBridge && (!previousProvider || !previousModel)) { + throw new InferenceSetError( + `Cannot verify the sandbox-only provider route because sandbox '${sandboxName}' does not record ` + + "the previous provider and model needed to restore its OpenShell inference selection.", + 2, + ); + } let appliedProvider = false; let appliedInferenceSelection = false; let restoredSelectionAfterProviderFailure = false; let providerMutation: ReturnType | null = null; + const restorePreviousInferenceSelection = (): string | null => { + let restoreResult: CaptureOpenshellResult; + try { + restoreResult = deps.captureOpenshell( + openshellInferenceSetArgs({ + gatewayName: preparedRoute.gatewayName, + provider: previousProvider, + model: previousModel, + noVerify: true, + }), + { + ignoreError: true, + includeStreams: true, + maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, + }, + ); + } catch { + return "the restore command could not be invoked"; + } + if (restoreResult.status !== 0) { + return `the restore command exited with status ${restoreResult.status ?? "unknown"}`; + } + appliedInferenceSelection = false; + return null; + }; try { const providerBinding = httpsPinProviderBinding ?? directProviderBinding; if (providerBinding) { @@ -1081,28 +1123,15 @@ async function runInferenceSetWithoutHostLock( providerError instanceof Error ? providerError.message : String(providerError); const providerExitCode = providerError instanceof InferenceSetError ? providerError.exitCode : 1; - const restoreResult = deps.captureOpenshell( - openshellInferenceSetArgs({ - gatewayName: preparedRoute.gatewayName, - provider: previousProvider, - model: previousModel, - noVerify: true, - }), - { - ignoreError: true, - includeStreams: true, - maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, - }, - ); - if (restoreResult.status !== 0) { + const restoreFailure = restorePreviousInferenceSelection(); + if (restoreFailure) { throw new InferenceSetError( `${providerDetail}\n Failed to restore the previous OpenShell inference selection ` + - `'${previousProvider}' / '${previousModel}' (status ${restoreResult.status ?? "unknown"}). ` + + `'${previousProvider}' / '${previousModel}': ${restoreFailure}. ` + `The live selection and provider binding may be split; re-run onboarding before using this route.`, providerExitCode, ); } - appliedInferenceSelection = false; restoredSelectionAfterProviderFailure = true; throw new InferenceSetError( `${providerDetail}\n The previous OpenShell inference selection was restored to ` + @@ -1113,6 +1142,44 @@ async function runInferenceSetWithoutHostLock( } } + if (probeDirectSandboxBridge) { + let probe: ReturnType; + try { + probe = deps.probeSandboxRoute({ + sandboxName, + provider, + model, + preferredInferenceApi: preMutationInferenceApi, + }); + } catch (probeError) { + const probeFailureDetail = + probeError instanceof Error && probeError.message + ? (onboardSession.redactSensitiveText(probeError.message)?.trim() ?? "") + : ""; + probe = { + ok: false, + detail: probeFailureDetail + ? `sandbox inference invocation probe was unavailable: ${probeFailureDetail}` + : "sandbox inference invocation probe was unavailable", + httpStatus: null, + }; + } + if (!probe.ok) { + const restoreFailure = restorePreviousInferenceSelection(); + if (restoreFailure) { + throw new InferenceSetError( + `Sandbox-side verification rejected provider '${provider}' / '${model}': ${probe.detail}. ` + + `Failed to restore the previous OpenShell inference selection '${previousProvider}' / ` + + `'${previousModel}': ${restoreFailure}. Re-run onboarding before using this route.`, + ); + } + throw new InferenceSetError( + `Sandbox-side verification rejected provider '${provider}' / '${model}': ${probe.detail}. ` + + `The previous OpenShell inference selection was restored to '${previousProvider}' / '${previousModel}'.`, + ); + } + } + // Write minimal registry state before any sandbox-facing config read so the // gateway and registry cannot split if the in-sandbox layer is unavailable. const registryFields = (preferredInferenceApi: string | null) => diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index d266f5d3ab..2f575e915d 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1901,7 +1901,7 @@ async function createSandboxWithBaseImageResolution( const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); const { initialSandboxPolicy, policyTier: resolvedCreatePolicyTier, messagingProviders, gpuRoutePlan, compatibilityPolicyPath, initialGpuRoute, sandboxReadyTimeoutSecs, buildId, dashboardRemoteBindPrepared, legacyBuildContext, launch: { createArgv, effectiveDashboardPort, intendedSandboxStartupCommand, managedBootstrapIdentity, managedStartupRootApplyRequest, prebuild, sandboxEnv, sandboxStartupCommand } } = await managedWorkloadOnboard.prepareOnboardSandboxWorkloadLaunch({ runtime: managedWorkloadRuntime, workload: preparedSandboxWorkload, - legacy: { preparedBuildContext, agent, fromDockerfile, createAgentSandbox: (selectedAgent) => baseImageResolutionFlow.createAgentSandboxWithResolution(baseImageResolutionContext, selectedAgent, agentOnboard.createAgentSandbox), patchInput: { preparedBuildContext, agent, fromDockerfile, model, chatUiUrl, provider, endpointUrl: createIntent?.endpointUrl ?? null, compatibleEndpointReasoning: createIntent?.compatibleEndpointReasoning, preferredInferenceApi, webSearchConfig, toolDisclosure: effectiveToolDisclosure, rebuildPreservedEnv: createIntent?.rebuildPreservedEnv, ...(isManagedDcodeAgent ? { dcodeAutoApprovalMode: dcodeAutoApprovalPlan.mode } : {}), hermesToolGateways, sandboxGpuConfig: effectiveSandboxGpuConfig, ...baseImageResolutionFlow.getBaseImageResolutionPatchOptions(baseImageResolutionContext), gatewayPort: GATEWAY_PORT } }, + legacy: { preparedBuildContext, agent, fromDockerfile, createAgentSandbox: (selectedAgent) => baseImageResolutionFlow.createAgentSandboxWithResolution(baseImageResolutionContext, selectedAgent, agentOnboard.createAgentSandbox), resolvePatchInput: () => ({ preparedBuildContext, agent, fromDockerfile, model, chatUiUrl, provider, endpointUrl: createIntent?.endpointUrl ?? null, compatibleEndpointReasoning: createIntent?.compatibleEndpointReasoning, preferredInferenceApi, webSearchConfig, toolDisclosure: effectiveToolDisclosure, rebuildPreservedEnv: createIntent?.rebuildPreservedEnv, ...(isManagedDcodeAgent ? { dcodeAutoApprovalMode: dcodeAutoApprovalPlan.mode } : {}), hermesToolGateways, sandboxGpuConfig: effectiveSandboxGpuConfig, ...baseImageResolutionFlow.getBaseImageResolutionPatchOptions(baseImageResolutionContext), gatewayPort: GATEWAY_PORT }) }, plan: { intent: resolvedCreateIntent, rebindMessagingTokenDefs: async () => (await sandboxCreateIntentResolver.rebind({ sandboxName, enabledChannels, webSearchConfig, agent, ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}) }, resolvedCreateIntent)).messagingTokenDefs, runProviderPreDeleteCleanup: () => runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact, tolerateMissingSandbox: true }), upsertMessagingProviders, getHermesToolGatewayProviderName: (targetSandbox) => getHermesToolGatewayBroker().getHermesToolGatewayProviderName(targetSandbox), discloseInitialSandboxPolicy }, launchInput: { agent, observabilityEnabled: createIntent?.observabilityEnabled === true, chatUiUrl, sandboxName, env: process.env, extraPlaceholderKeys: resolvedCreateIntent.extraPlaceholderKeys, getDashboardForwardPort, hermesDashboardState, hermesApiPort: hermesApiPortReservationScope.effectivePort, manageDashboard, openshellShellCommand, openshellArgv }, plannedMessagingPlan: plannedMessagingState?.plan ?? null, diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts new file mode 100644 index 0000000000..d9b3d312ae --- /dev/null +++ b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { prepareOnboardSandboxWorkloadLaunch } from "./onboard-orchestration"; + +describe("managed workload onboard orchestration", () => { + it("resolves final-image patch metadata after managed build-context staging", async () => { + const resolutionMetadata = { key: "published-dcode-base" }; + let staged = false; + const resolvePatchInput = vi.fn(() => { + expect(staged).toBe(true); + return { preResolvedBaseImageMetadata: resolutionMetadata } as never; + }); + const resolveSandboxBuildPatch = vi.fn(async (input: Record) => { + expect(input.preResolvedBaseImageMetadata).toBe(resolutionMetadata); + expect(input.stagedDockerfile).toBe("/tmp/nemoclaw-staged-context/Dockerfile"); + return { buildId: "dcode-build", dashboardRemoteBindPrepared: false }; + }); + const materializeSandboxCreatePlan = vi.fn(() => ({ + activeMessagingChannels: [], + compatibilityPolicyPath: null, + createArgs: [ + "--from", + "/tmp/nemoclaw-staged-context/Dockerfile", + "--name", + "dcode", + "--policy", + "/tmp/nemoclaw-policy.yaml", + ], + gpuRoutePlan: "none", + initialSandboxPolicy: { + appliedPresets: [], + policyPath: "/tmp/nemoclaw-policy.yaml", + }, + messagingProviders: [], + policyTier: null, + sandboxGpuLogMessage: null, + })); + + await prepareOnboardSandboxWorkloadLaunch({ + runtime: { + runtimeProvider: null, + ensurePreparedWorkload: vi.fn(), + ensurePreparedProfile: vi.fn(), + }, + workload: { + source: { + kind: "legacy-dockerfile", + dockerfilePath: "agents/langchain-deepagents-code/Dockerfile", + reason: "runtime-unsupported", + }, + release: "v0.0.0", + fallbackDiagnostic: null, + }, + legacy: { + preparedBuildContext: null, + agent: { + name: "langchain-deepagents-code", + displayName: "LangChain Deep Agents Code", + }, + fromDockerfile: null, + createAgentSandbox: () => { + staged = true; + return { + buildCtx: "/tmp/nemoclaw-staged-context", + stagedDockerfile: "/tmp/nemoclaw-staged-context/Dockerfile", + baseImageResolutionMetadata: resolutionMetadata, + }; + }, + resolvePatchInput, + }, + plan: { + intent: {}, + rebindMessagingTokenDefs: async () => [], + runProviderPreDeleteCleanup: vi.fn(), + upsertMessagingProviders: vi.fn(() => []), + getHermesToolGatewayProviderName: vi.fn(() => "unused"), + discloseInitialSandboxPolicy: vi.fn(), + }, + launchInput: { + agent: null, + chatUiUrl: "http://127.0.0.1:18789", + sandboxName: "dcode", + env: { NEMOCLAW_SANDBOX_PREBUILD: "0" }, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: {}, + manageDashboard: false, + openshellShellCommand: () => "openshell sandbox create", + }, + plannedMessagingPlan: null, + gpu: { + provider: "compatible-endpoint", + config: { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }, + dockerDriverGateway: false, + gatewayPort: 8080, + }, + dependencies: { + materializeSandboxCreatePlan, + prepareSandboxBuildPatchConfig: vi.fn(() => ({ + messagingChannelConfig: null, + })), + resolveSandboxBuildPatch, + }, + } as unknown as Parameters[0]); + + expect(resolvePatchInput).toHaveBeenCalledOnce(); + expect(resolveSandboxBuildPatch).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index e28c25ef83..8ebcdd25df 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -245,7 +245,10 @@ export interface PrepareOnboardSandboxWorkloadLaunchInput { readonly createAgentSandbox: ( agent: AgentDefinition, ) => ReturnType; - readonly patchInput: Omit; + readonly resolvePatchInput: () => Omit< + ResolveBuildPatchInput, + "selectedGpuRoute" | "stagedDockerfile" + >; }; readonly plan: { readonly intent: SandboxCreateIntent; @@ -269,6 +272,7 @@ export interface PrepareOnboardSandboxWorkloadLaunchInput { readonly dependencies: { readonly materializeSandboxCreatePlan: typeof import("../sandbox-create-plan-materialization").materializeSandboxCreatePlan; readonly prepareSandboxBuildPatchConfig: typeof import("../sandbox-build-patch-config").prepareSandboxBuildPatchConfig; + readonly resolveSandboxBuildPatch?: typeof import("../prepared-dcode-rebuild").resolveSandboxBuildPatch; }; readonly log?: (message: string) => void; readonly onExit?: (cleanup: () => void) => void; @@ -377,8 +381,13 @@ export async function prepareOnboardSandboxWorkloadLaunch( } else { const buildContext = requireLegacyBuildContext(legacyBuildContext); input.dependencies.prepareSandboxBuildPatchConfig({ configuredMessagingChannels }); - const patch = await resolveSandboxBuildPatch({ - ...input.legacy.patchInput, + const patch = await ( + input.dependencies.resolveSandboxBuildPatch ?? resolveSandboxBuildPatch + )({ + // Build-context staging resolves managed-agent base-image provenance. + // Read the patch input only after that boundary so the final image gets + // the exact metadata produced by the same staging operation. + ...input.legacy.resolvePatchInput(), selectedGpuRoute: initialGpuRoute, stagedDockerfile: buildContext.stagedDockerfile, }); diff --git a/test/e2e/fixtures/compatible-anthropic-switch.ts b/test/e2e/fixtures/compatible-anthropic-switch.ts index d80c96d29d..59d019ad81 100644 --- a/test/e2e/fixtures/compatible-anthropic-switch.ts +++ b/test/e2e/fixtures/compatible-anthropic-switch.ts @@ -1,13 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { randomBytes } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { parseDockerDriverGatewayRuntimeMarker } from "../../../src/lib/onboard/docker-driver-gateway-runtime-marker.ts"; -import { resolveDockerDriverGatewayStateDir } from "../../../src/lib/onboard/host-gateway-process.ts"; import type { HostCliClient } from "./clients/host.ts"; import { resultText } from "./clients/index.ts"; @@ -15,71 +8,6 @@ export const COMPATIBLE_ANTHROPIC_PROVIDER = "compatible-anthropic-endpoint"; export const COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV = "COMPATIBLE_ANTHROPIC_API_KEY"; const DEFAULT_COMPATIBLE_ANTHROPIC_CREDENTIAL = "test-compatible-anthropic-key"; const OPENSHELL_HOST_ALIAS = "host.openshell.internal"; -const GATEWAY_SERVICE_NAMES = ["nemoclaw-openshell-gateway", "openshell-gateway"] as const; -const GATEWAY_STATE_FILE_LIMIT = 64 * 1024; - -export const GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT = [ - "set -euo pipefail", - 'operation="$1"', - 'resolver_source="$2"', - 'owner_token="$3"', - 'hosts_path="$4"', - `alias_name="${OPENSHELL_HOST_ALIAS}"`, - 'owned_line="127.0.0.1 ${alias_name} # nemoclaw-gateway-host-verifier:${owner_token}"', - "", - 'case "$operation" in', - " add | remove) ;;", - ' *) echo "unsupported gateway resolver operation: $operation" >&2; exit 2 ;;', - "esac", - '[[ "$owner_token" =~ ^[a-f0-9]{32}$ ]] || { echo "invalid gateway resolver owner token" >&2; exit 2; }', - '[[ -f "$hosts_path" ]] || { echo "gateway resolver path is not a regular file" >&2; exit 2; }', - "", - 'if [[ "$operation" == "add" ]]; then', - ' [[ -f "$resolver_source" && ! -L "$resolver_source" ]] || { echo "gateway resolver source is not a regular file" >&2; exit 2; }', - ' grep -Fqx -- "$owned_line" "$resolver_source" || { echo "gateway resolver source lacks its ownership marker" >&2; exit 2; }', - " mount --make-rprivate /", - ' mount --bind "$resolver_source" "$hosts_path"', - ' grep -Fqx -- "$owned_line" "$hosts_path" || { echo "gateway resolver mount was not installed" >&2; exit 4; }', - " exit 0", - "fi", - "", - '# A restarted gateway has already released the owned mount namespace.', - 'grep -Fqx -- "$owned_line" "$hosts_path" || exit 0', - 'umount "$hosts_path"', - 'if grep -Fqx -- "$owned_line" "$hosts_path"; then', - ' echo "gateway resolver mount was not removed" >&2', - " exit 4", - "fi", -].join("\n"); - -const GATEWAY_HOST_VERIFICATION_NAMESPACE_SCRIPT = [ - "set -euo pipefail", - 'operation="$1"', - 'target_pid="$2"', - 'resolver_source="$3"', - 'owner_token="$4"', - 'mount_script="$5"', - "", - '[[ "$target_pid" =~ ^[1-9][0-9]*$ ]] || { echo "invalid OpenShell gateway PID" >&2; exit 2; }', - "gateway_is_alive() {", - ' local executable=""', - ' [[ -r "/proc/${target_pid}/stat" ]] || return 1', - ' executable="$(readlink -f "/proc/${target_pid}/exe" 2>/dev/null || true)"', - ' [[ "${executable##*/}" == "openshell-gateway" ]]', - "}", - "", - 'if ! gateway_is_alive; then', - ' [[ "$operation" == "remove" ]] && exit 0', - ' echo "active OpenShell gateway process is unavailable" >&2', - " exit 3", - "fi", - 'command -v nsenter >/dev/null 2>&1 || { echo "nsenter is required for scoped gateway resolution" >&2; exit 2; }', - 'current_namespace="$(readlink /proc/self/ns/mnt)"', - 'target_namespace="$(readlink "/proc/${target_pid}/ns/mnt")"', - '[[ "$current_namespace" != "$target_namespace" ]] || { echo "OpenShell gateway does not have a private mount namespace" >&2; exit 3; }', - "", - 'exec nsenter --target "$target_pid" --mount -- bash -ceu "$mount_script" gateway-resolver-mount "$operation" "$resolver_source" "$owner_token" /etc/hosts', -].join("\n"); export interface CompatibleAnthropicSwitchBinding { endpointUrl: string; @@ -90,189 +18,6 @@ export function compatibleAnthropicMockEndpointUrl(port: number): string { return `http://${OPENSHELL_HOST_ALIAS}:${port}`; } -function pathExists(filePath: string): boolean { - try { - fs.lstatSync(filePath); - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; - throw error; - } -} - -function readOwnedGatewayStateFile(filePath: string, currentUid: number): string | null { - if (typeof fs.constants.O_NOFOLLOW !== "number") return null; - let descriptor: number | undefined; - try { - descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); - const stat = fs.fstatSync(descriptor); - if ( - !stat.isFile() || - stat.nlink !== 1 || - stat.uid !== currentUid || - (stat.mode & 0o022) !== 0 || - stat.size > GATEWAY_STATE_FILE_LIMIT - ) { - return null; - } - return fs.readFileSync(descriptor, "utf8"); - } catch { - return null; - } finally { - if (descriptor !== undefined) fs.closeSync(descriptor); - } -} - -function managedOpenShellGatewayPid(homeDir: string): number | null { - const stateDirectory = resolveDockerDriverGatewayStateDir(process.env, homeDir); - const pidPath = path.join(stateDirectory, "openshell-gateway.pid"); - const markerPath = path.join(stateDirectory, "runtime.json"); - const pidPathExists = pathExists(pidPath); - const markerPathExists = pathExists(markerPath); - if (!pidPathExists && !markerPathExists) return null; - if (!pidPathExists || !markerPathExists) { - throw new Error("Docker-driver gateway state is incomplete"); - } - - const currentUid = process.getuid?.(); - if (currentUid === undefined) { - throw new Error("Docker-driver gateway state ownership is unavailable"); - } - const pidText = readOwnedGatewayStateFile(pidPath, currentUid); - const markerText = readOwnedGatewayStateFile(markerPath, currentUid); - if (!pidText || !markerText) { - throw new Error("Docker-driver gateway state is not an owned regular file"); - } - if (!/^[1-9][0-9]*\n?$/u.test(pidText)) { - throw new Error("Docker-driver gateway PID file is invalid"); - } - const pid = Number(pidText.trim()); - const marker = parseDockerDriverGatewayRuntimeMarker(markerText); - if ( - !Number.isSafeInteger(pid) || - !marker || - marker.pid !== pid || - marker.platform !== process.platform || - marker.arch !== process.arch - ) { - throw new Error("Docker-driver gateway state does not identify the current process"); - } - if (marker.endpoint !== "https://127.0.0.1:8080") { - throw new Error("Docker-driver gateway state does not identify the default gateway"); - } - - let processStat: fs.Stats; - let executable: string; - try { - processStat = fs.statSync(`/proc/${pid}`); - executable = fs.realpathSync(`/proc/${pid}/exe`); - } catch { - throw new Error("Docker-driver gateway process is unavailable"); - } - if (processStat.uid !== currentUid || path.basename(executable) !== "openshell-gateway") { - throw new Error("Docker-driver gateway process identity does not match its state"); - } - if (marker.gatewayBin) { - let recordedExecutable: string; - try { - recordedExecutable = fs.realpathSync(marker.gatewayBin); - } catch { - throw new Error("Docker-driver gateway executable is unavailable"); - } - if (executable !== recordedExecutable) { - throw new Error("Docker-driver gateway executable does not match its state"); - } - } - return pid; -} - -async function activeOpenShellGatewayPid(host: HostCliClient, homeDir: string): Promise { - const managedPid = managedOpenShellGatewayPid(homeDir); - if (managedPid !== null) return managedPid; - for (const serviceName of GATEWAY_SERVICE_NAMES) { - const result = await host.command( - "systemctl", - [ - "--user", - "show", - serviceName, - "--property=ActiveState", - "--property=MainPID", - ], - { artifactName: `inspect-${serviceName}`, timeoutMs: 30_000 }, - ); - if (result.exitCode !== 0) continue; - const properties = new Map( - result.stdout - .split(/\r?\n/u) - .map((line) => line.split("=", 2)) - .filter((entry): entry is [string, string] => entry.length === 2), - ); - const pid = Number(properties.get("MainPID")); - if (properties.get("ActiveState") === "active" && Number.isSafeInteger(pid) && pid > 0) { - return pid; - } - } - throw new Error("could not find an active OpenShell gateway user service"); -} - -export async function installGatewayHostVerificationAlias( - host: HostCliClient, - cleanup: { add(name: string, run: () => Promise | void): void }, - homeDir: string = os.homedir(), -): Promise { - const gatewayPid = await activeOpenShellGatewayPid(host, homeDir); - const ownerToken = randomBytes(16).toString("hex"); - const fixtureDirectory = fs.mkdtempSync( - path.join(homeDir, ".nemoclaw-gateway-resolver-"), - ); - const resolverSource = path.join(fixtureDirectory, "hosts"); - const ownedLine = `127.0.0.1 ${OPENSHELL_HOST_ALIAS} # nemoclaw-gateway-host-verifier:${ownerToken}`; - fs.chmodSync(fixtureDirectory, 0o700); - fs.writeFileSync(resolverSource, `${ownedLine}\n${fs.readFileSync("/etc/hosts", "utf8")}`, { - mode: 0o600, - }); - - const updateMount = async (operation: "add" | "remove"): Promise => { - const result = await host.command( - "sudo", - [ - "bash", - "-ceu", - GATEWAY_HOST_VERIFICATION_NAMESPACE_SCRIPT, - `gateway-resolver-${operation}`, - operation, - String(gatewayPid), - resolverSource, - ownerToken, - GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT, - ], - { artifactName: `${operation}-gateway-host-verifier-alias`, timeoutMs: 60_000 }, - ); - if (result.exitCode !== 0) { - throw new Error( - `could not ${operation === "add" ? "install" : "remove"} the gateway host verifier alias: ${resultText(result)}`, - ); - } - }; - - let restored = false; - const restore = async (): Promise => { - if (restored) return; - await updateMount("remove"); - restored = true; - fs.rmSync(fixtureDirectory, { force: true, recursive: true }); - }; - cleanup.add("remove the OpenShell gateway resolver mount", restore); - - try { - await updateMount("add"); - } catch (error) { - await restore(); - throw error; - } -} - export function compatibleAnthropicSwitchBinding( endpointUrl: string, runtimeEnv: NodeJS.ProcessEnv = process.env, diff --git a/test/e2e/live/hermes-inference-switch-helpers.ts b/test/e2e/live/hermes-inference-switch-helpers.ts index d7aff6030a..f485101de4 100644 --- a/test/e2e/live/hermes-inference-switch-helpers.ts +++ b/test/e2e/live/hermes-inference-switch-helpers.ts @@ -21,7 +21,6 @@ import { compatibleAnthropicMockEndpointUrl, compatibleAnthropicSwitchBinding, compatibleAnthropicSwitchEnv, - installGatewayHostVerificationAlias, requireCompatibleAnthropicProviderAbsent, } from "../fixtures/compatible-anthropic-switch.ts"; import { expect } from "../fixtures/e2e-test.ts"; @@ -611,7 +610,6 @@ export async function prepareCompatibleAnthropicSwitchBinding( return null; const mock = mockAnthropicSwitchEnabled() ? await startMockAnthropicProvider() : undefined; mock && cleanup.add("close compatible Anthropic switch mock", () => mock.close()); - if (mock) await installGatewayHostVerificationAlias(host, cleanup); const binding = compatibleAnthropicSwitchBinding( process.env.NEMOCLAW_SWITCH_ENDPOINT_URL ?? mock?.endpointUrl ?? "", ); diff --git a/test/e2e/live/openclaw-inference-switch.test.ts b/test/e2e/live/openclaw-inference-switch.test.ts index 298d3c9356..2e81df9fc1 100644 --- a/test/e2e/live/openclaw-inference-switch.test.ts +++ b/test/e2e/live/openclaw-inference-switch.test.ts @@ -30,7 +30,6 @@ import { compatibleAnthropicMockEndpointUrl, compatibleAnthropicSwitchBinding, compatibleAnthropicSwitchEnv, - installGatewayHostVerificationAlias, requireCompatibleAnthropicProviderAbsent, } from "../fixtures/compatible-anthropic-switch.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -1093,7 +1092,6 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha if (SWITCH_PROVIDER === "compatible-anthropic-endpoint" && SWITCH_MOCK_ANTHROPIC === "1") { mockProvider = await startMockAnthropicProvider(); - await installGatewayHostVerificationAlias(host, cleanup, home); await artifacts.writeJson("mock-anthropic-provider.json", { endpointUrl: mockProvider.endpointUrl, }); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index bbfec6b1fa..25d7feac17 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -364,6 +364,8 @@ { "live": "test/e2e/live/hermes-inference-switch.test.ts", "fast": [ + "src/lib/actions/inference-set-compatible-provider.test.ts", + "test/e2e/support/compatible-anthropic-switch.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -448,6 +450,8 @@ { "live": "test/e2e/live/openclaw-inference-switch.test.ts", "fast": [ + "src/lib/actions/inference-set-compatible-provider.test.ts", + "test/e2e/support/compatible-anthropic-switch.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] diff --git a/test/e2e/support/compatible-anthropic-switch.test.ts b/test/e2e/support/compatible-anthropic-switch.test.ts index 0c56155985..2afeb72495 100644 --- a/test/e2e/support/compatible-anthropic-switch.test.ts +++ b/test/e2e/support/compatible-anthropic-switch.test.ts @@ -1,73 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - import { afterEach, describe, expect, it, vi } from "vitest"; import { normalizeCustomEndpointUrl } from "../../../src/lib/actions/inference-set.ts"; -import { - writeDockerDriverGatewayPidFile, - writeDockerDriverGatewayRuntimeMarkerForStateDir, -} from "../../../src/lib/onboard/docker-driver-gateway-runtime-marker.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { COMPATIBLE_ANTHROPIC_CREDENTIAL_ENV, COMPATIBLE_ANTHROPIC_PROVIDER, - GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT, compatibleAnthropicMockEndpointUrl, compatibleAnthropicSwitchBinding, compatibleAnthropicSwitchEnv, - installGatewayHostVerificationAlias, requireCompatibleAnthropicProviderAbsent, } from "../fixtures/compatible-anthropic-switch.ts"; -const INVALID_MANAGED_GATEWAY_STATE_CASES = [ - { - label: "invalid", - pid: process.pid, - writePid: (stateDirectory: string, _pid: number) => - fs.writeFileSync(path.join(stateDirectory, "openshell-gateway.pid"), "not-a-pid\n", { - mode: 0o600, - }), - }, - { - label: "symlinked", - pid: process.pid, - writePid: (stateDirectory: string, pid: number) => { - const target = path.join(stateDirectory, "pid-target"); - fs.writeFileSync(target, `${pid}\n`, { mode: 0o600 }); - fs.symlinkSync(target, path.join(stateDirectory, "openshell-gateway.pid")); - }, - }, - { - label: "stale", - pid: 2_147_483_647, - writePid: (stateDirectory: string, pid: number) => - writeDockerDriverGatewayPidFile( - path.join(stateDirectory, "openshell-gateway.pid"), - pid, - ), - }, -] as const; - -function mockGatewayProcess(pid: number, gatewayBin: string): void { - const statSync = fs.statSync; - vi.spyOn(fs, "statSync").mockImplementation(((target) => - String(target) === `/proc/${pid}` - ? ({ uid: process.getuid?.() ?? 0 } as fs.Stats) - : statSync(target)) as typeof fs.statSync); - const realpathSync = fs.realpathSync; - const gatewayExecutablePaths = new Set([`/proc/${pid}/exe`, gatewayBin]); - vi.spyOn(fs, "realpathSync").mockImplementation(((target) => - gatewayExecutablePaths.has(String(target)) - ? gatewayBin - : realpathSync(target)) as typeof fs.realpathSync); -} - describe("compatible Anthropic inference switch setup", () => { afterEach(() => { vi.unstubAllEnvs(); @@ -113,127 +59,6 @@ describe("compatible Anthropic inference switch setup", () => { expect(rewrite).not.toHaveBeenCalled(); }); - it("uses managed Docker-driver gateway state from the target home before the user service (#9166)", async () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-target-home-gateway-test-")); - const stateDirectory = path.join( - home, - ".local", - "state", - "nemoclaw", - "openshell-docker-gateway", - ); - const pid = process.pid; - const gatewayBin = "/usr/bin/openshell-gateway"; - fs.mkdirSync(stateDirectory, { recursive: true }); - vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", ""); - writeDockerDriverGatewayPidFile(path.join(stateDirectory, "openshell-gateway.pid"), pid); - writeDockerDriverGatewayRuntimeMarkerForStateDir(stateDirectory, { - desiredEnv: {}, - endpoint: "https://127.0.0.1:8080", - gatewayBin, - pid, - }); - mockGatewayProcess(pid, gatewayBin); - const command = vi.fn().mockResolvedValue({ exitCode: 0, stderr: "", stdout: "" }); - const add = vi.fn(); - - try { - await installGatewayHostVerificationAlias( - { command } as unknown as HostCliClient, - { add }, - home, - ); - const cleanupMount = add.mock.calls[0]?.[1] as () => Promise; - await cleanupMount(); - - expect(command).toHaveBeenCalledTimes(2); - for (const call of command.mock.calls) { - expect(call[0]).toBe("sudo"); - expect(call[1]).toEqual( - expect.arrayContaining([String(pid), GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT]), - ); - } - } finally { - fs.rmSync(home, { force: true, recursive: true }); - } - }); - - it("uses the active user service when managed gateway state is absent (#9166)", async () => { - const stateDirectory = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-missing-gateway-state-test-"), - ); - vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", stateDirectory); - const command = vi - .fn() - .mockResolvedValueOnce({ - exitCode: 0, - stderr: "", - stdout: "ActiveState=active\nMainPID=4242\n", - }) - .mockResolvedValue({ exitCode: 0, stderr: "", stdout: "" }); - const add = vi.fn(); - - try { - await installGatewayHostVerificationAlias( - { command } as unknown as HostCliClient, - { add }, - stateDirectory, - ); - const cleanupMount = add.mock.calls[0]?.[1] as () => Promise; - await cleanupMount(); - - expect(command.mock.calls[0]?.slice(0, 2)).toEqual([ - "systemctl", - [ - "--user", - "show", - "nemoclaw-openshell-gateway", - "--property=ActiveState", - "--property=MainPID", - ], - ]); - for (const call of command.mock.calls.slice(1)) { - expect(call[0]).toBe("sudo"); - expect(call[1]).toEqual( - expect.arrayContaining(["4242", GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT]), - ); - } - } finally { - fs.rmSync(stateDirectory, { force: true, recursive: true }); - } - }); - - it.each(INVALID_MANAGED_GATEWAY_STATE_CASES)( - "rejects $label managed gateway PID state (#9166)", - async ({ pid, writePid }) => { - const stateDirectory = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-invalid-gateway-state-test-"), - ); - vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", stateDirectory); - writeDockerDriverGatewayRuntimeMarkerForStateDir(stateDirectory, { - desiredEnv: {}, - endpoint: "https://127.0.0.1:8080", - gatewayBin: "/usr/bin/openshell-gateway", - pid, - }); - writePid(stateDirectory, pid); - const command = vi.fn(); - - try { - await expect( - installGatewayHostVerificationAlias({ command } as unknown as HostCliClient, { - add: vi.fn(), - }), - ).rejects.toThrow( - /Docker-driver gateway (PID file is invalid|process is unavailable|state is not an owned regular file)/u, - ); - expect(command).not.toHaveBeenCalled(); - } finally { - fs.rmSync(stateDirectory, { force: true, recursive: true }); - } - }, - ); - it("requires the direct provider to be absent before inference set owns its creation", async () => { const command = vi.fn().mockResolvedValue({ exitCode: 1, @@ -283,80 +108,3 @@ describe("compatible Anthropic inference switch setup", () => { ); }); }); - -const linuxIt = process.platform === "linux" ? it : it.skip; - -describe("gateway resolver mount", () => { - linuxIt("preserves a resolver write that overlaps mount installation (#9166)", () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-resolver-test-")); - const hostsPath = path.join(directory, "hosts"); - const underlayPath = path.join(directory, "hosts.underlay"); - const resolverSource = path.join(directory, "resolver-source"); - const fakeBin = path.join(directory, "bin"); - const token = "a".repeat(32); - const ownedLine = - `127.0.0.1 host.openshell.internal # nemoclaw-gateway-host-verifier:${token}`; - try { - fs.mkdirSync(fakeBin); - fs.writeFileSync(hostsPath, "127.0.0.1 localhost\n", { mode: 0o644 }); - fs.writeFileSync(resolverSource, `${ownedLine}\n127.0.0.1 localhost\n`, { mode: 0o600 }); - fs.writeFileSync( - path.join(fakeBin, "mount"), - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - '[[ "$1" == "--make-rprivate" ]] && exit 0', - '[[ "$1" == "--bind" ]]', - "printf '192.0.2.10 concurrent.example.test\\n' >> \"$3\"", - 'mv -- "$3" "$NEMOCLAW_TEST_RESOLVER_UNDERLAY"', - 'ln -s -- "$2" "$3"', - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(fakeBin, "umount"), - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - 'rm -- "$1"', - 'mv -- "$NEMOCLAW_TEST_RESOLVER_UNDERLAY" "$1"', - ].join("\n"), - { mode: 0o755 }, - ); - const run = (operation: "add" | "remove") => - spawnSync( - "bash", - [ - "-ceu", - GATEWAY_HOST_VERIFICATION_MOUNT_SCRIPT, - "gateway-resolver-mount-test", - operation, - resolverSource, - token, - hostsPath, - ], - { - encoding: "utf8", - env: { - ...process.env, - NEMOCLAW_TEST_RESOLVER_UNDERLAY: underlayPath, - PATH: `${fakeBin}:${process.env.PATH ?? ""}`, - }, - }, - ); - - const added = run("add"); - expect(added.status, added.stderr).toBe(0); - expect(fs.readFileSync(hostsPath, "utf8")).toContain(ownedLine); - expect(fs.readFileSync(underlayPath, "utf8")).toContain("concurrent.example.test"); - - const removed = run("remove"); - expect(removed.status, removed.stderr).toBe(0); - expect(fs.readFileSync(hostsPath, "utf8")).toBe( - "127.0.0.1 localhost\n192.0.2.10 concurrent.example.test\n", - ); - } finally { - fs.rmSync(directory, { force: true, recursive: true }); - } - }); -});