diff --git a/ci/e2e-assertion-budget.json b/ci/e2e-assertion-budget.json index 8aa3e1e4709..17a045104e2 100644 --- a/ci/e2e-assertion-budget.json +++ b/ci/e2e-assertion-budget.json @@ -15,14 +15,14 @@ "testFileCount": 86, "liveFileCount": 222, "direct": { - "expectCalls": 1800, - "matcherAssertions": 1769, + "expectCalls": 1798, + "matcherAssertions": 1767, "nodeAssertions": 105, "namedAssertionHelpers": 601, "failCalls": 8, "throwGuards": 87, "objectFieldAssertions": 243, - "assertionPoints": 2813, + "assertionPoints": 2811, "generatedProbeBlocks": 122, "generatedProbeConditions": 322 }, @@ -49,7 +49,7 @@ "test/e2e/live/agent-turn-latency.test.ts": [22,26,31,46,1], "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts": [28,64,28,68,5], "test/e2e/live/bootstrap-install-smoke.test.ts": [0,0,14,30,3], - "test/e2e/live/brave-search.test.ts": [19,22,27,32,1], + "test/e2e/live/brave-search.test.ts": [17,20,27,32,1], "test/e2e/live/brev-workspace-cleanup.test.ts": [0,0,0,1,0], "test/e2e/live/channels-add-remove.test.ts": [40,93,40,93,0], "test/e2e/live/channels-stop-start.test.ts": [0,0,36,130,9], diff --git a/schemas/nemoclaw-config-v1.schema.json b/schemas/nemoclaw-config-v1.schema.json index d93e08cff89..0a5c26f4998 100644 --- a/schemas/nemoclaw-config-v1.schema.json +++ b/schemas/nemoclaw-config-v1.schema.json @@ -56,42 +56,170 @@ "inferenceProviders": { "type": "array", "items": { - "type": "object", - "required": ["name", "provider", "api", "endpoint"], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "maxLength": 63, - "pattern": "^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$" - }, - "provider": { - "type": "string", - "minLength": 1, - "maxLength": 512, - "pattern": "^[^\\s\\p{Cc}\\p{Cf}]+$" - }, - "api": { - "enum": ["openai-completions", "openai-responses", "anthropic-messages"] - }, - "endpoint": { - "type": "string", - "maxLength": 2048, - "pattern": "^https://[^\\s]+$" + "anyOf": [ + { + "type": "object", + "required": ["name", "provider", "api", "endpoint"], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 63, + "pattern": "^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$" + }, + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[^\\s\\p{Cc}\\p{Cf}]+$", + "not": { + "const": "vllm-local" + } + }, + "api": { + "enum": ["openai-completions", "openai-responses", "anthropic-messages"] + }, + "endpoint": { + "type": "string", + "maxLength": 2048, + "pattern": "^https://[^\\s]+$" + }, + "credential": { + "type": "object", + "required": ["env"], + "properties": { + "env": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_]{0,127}$" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false }, - "credential": { + { "type": "object", - "required": ["env"], + "required": ["name", "provider", "api", "serving"], "properties": { - "env": { + "name": { "type": "string", - "pattern": "^[A-Z][A-Z0-9_]{0,127}$" + "minLength": 1, + "maxLength": 63, + "pattern": "^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$" + }, + "provider": { + "type": "string", + "const": "vllm-local" + }, + "api": { + "type": "string", + "const": "openai-completions" + }, + "serving": { + "type": "object", + "required": [ + "backend", + "catalogDigest", + "profile", + "recipe", + "model", + "runtime", + "hostPort" + ], + "properties": { + "backend": { + "type": "string", + "const": "vllm" + }, + "catalogDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "profile": { + "type": "object", + "required": ["id", "digest"], + "properties": { + "id": { + "type": "string", + "const": "vllm.linux-amd64-nvidia.single.nemotron-3.5-lightning-30b-a3b-nvfp4" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + }, + "additionalProperties": false + }, + "recipe": { + "type": "object", + "required": ["id", "digest"], + "properties": { + "id": { + "type": "string", + "const": "vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.linux-amd64-single.v1" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + }, + "additionalProperties": false + }, + "model": { + "type": "object", + "required": ["id", "revision", "servedName"], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[^\\s\\p{Cc}\\p{Cf}]+$" + }, + "revision": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "servedName": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[^\\s\\p{Cc}\\p{Cf}]+$" + } + }, + "additionalProperties": false + }, + "runtime": { + "type": "object", + "required": ["image"], + "properties": { + "image": { + "type": "object", + "required": ["ref"], + "properties": { + "ref": { + "type": "string", + "maxLength": 512, + "pattern": "^(?:[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]+)?/)?(?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)*[a-z0-9]+(?:[._-][a-z0-9]+)*@sha256:[0-9a-f]{64}$" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "hostPort": { + "type": "integer", + "minimum": 1024, + "maximum": 65535 + } + }, + "additionalProperties": false } }, "additionalProperties": false } - }, - "additionalProperties": false + ] }, "minItems": 1 }, @@ -211,6 +339,11 @@ "minLength": 1, "maxLength": 512, "pattern": "^[^\\s\\p{Cc}\\p{Cf}]+$" + }, + "contextWindow": { + "type": "integer", + "minimum": 1, + "maximum": 4194304 } }, "additionalProperties": false @@ -227,6 +360,44 @@ "additionalProperties": false }, "minItems": 1 + }, + "integrations": { + "type": "object", + "required": ["webSearch"], + "properties": { + "webSearch": { + "type": "object", + "required": ["provider", "agentRefs", "credential"], + "properties": { + "provider": { + "type": "string", + "const": "brave" + }, + "agentRefs": { + "type": "array", + "items": { + "type": "string", + "const": "primary" + }, + "minItems": 1, + "maxItems": 1 + }, + "credential": { + "type": "object", + "required": ["env"], + "properties": { + "env": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_]{0,127}$" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/src/lib/actions/config/observe-export-source.ts b/src/lib/actions/config/observe-export-source.ts index cbf71e78476..1a76b25431b 100644 --- a/src/lib/actions/config/observe-export-source.ts +++ b/src/lib/actions/config/observe-export-source.ts @@ -84,6 +84,8 @@ const LIVE_READ_SOURCE_LABELS = { "sandbox-identity": "live sandbox identity", "inference-route": "live gateway inference route", "provider-metadata": "live inference provider metadata", + "web-search-provider": "live web-search provider metadata", + "managed-serving": "managed serving runtime", "effective-policy": "effective OpenShell policy", } satisfies Readonly>; diff --git a/src/lib/adapters/config/live-export-source.test.ts b/src/lib/adapters/config/live-export-source.test.ts index 07f0578ee23..a92cf53a523 100644 --- a/src/lib/adapters/config/live-export-source.test.ts +++ b/src/lib/adapters/config/live-export-source.test.ts @@ -3,13 +3,20 @@ import { describe, expect, it, vi } from "vitest"; import YAML from "yaml"; +import { managedBraveProfile } from "../../../../test/fixtures/openshell-provider-profile"; import { runConfigExport } from "../../actions/config/export"; import { parseNemoClawConfigDocumentName, + EXPORTED_VLLM_PROFILE_ID, + EXPORTED_VLLM_RECIPE_ID, + type ImmutableImageReference, parseNemoClawConfigDocumentUid, } from "../../config/model"; import { validateNemoClawConfig } from "../../config/schema"; +vi.mock("../../inference/serving/vllm-export-runtime", () => ({ + observeManagedVllmForExport: vi.fn(), +})); vi.mock("../../state/registry/persistence", () => ({ load: vi.fn() })); vi.mock("../../state/registry-entry-view", () => ({ getSandboxEntryInference: vi.fn() })); vi.mock("../../inference/live", () => ({ getLiveGatewayInference: vi.fn() })); @@ -30,9 +37,18 @@ vi.mock("../../onboard/gateway/state-dir", () => ({ resolveGatewayStateDirForPort: vi.fn(() => "/managed/gateway"), })); +import { observeManagedVllmForExport } from "../../inference/serving/vllm-export-runtime"; +import { loadServingCatalog } from "../../inference/serving/catalog-loader"; +import { servingProfileProvenance } from "../../inference/serving/profile-provenance"; +import { applyVllmRuntimeContextWindow } from "../../inference/vllm-runtime-context"; +import { resolveManagedStartupInferenceRoute } from "../../inference/gateway/route-contract"; +import type { ObservedManagedVllmRuntime } from "../../domain/config/export-evidence"; import { getLiveGatewayInference } from "../../inference/live"; import { resolveGatewayStateDirForPort } from "../../onboard/gateway/state-dir"; -import { buildManagedStartupProfile } from "../../onboard/managed-startup/profile-builder"; +import { + buildManagedStartupProfile, + type ManagedStartupProfileBuilderInput, +} from "../../onboard/managed-startup/profile-builder"; import { getSandboxEntryInference } from "../../state/registry-entry-view"; import { load as loadRegistry } from "../../state/registry/persistence"; import type { SandboxEntry } from "../../state/registry/types"; @@ -47,7 +63,7 @@ const identityFingerprint = fingerprintOpenShellSandboxId(sandboxId)!; const endpoint = "https://integrate.api.nvidia.com/v1"; const readFailureCanary = "credential-canary-value"; const imageRef = "ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:" + "a".repeat(64); -const startup = buildManagedStartupProfile({ +const startupInput = { agent: "openclaw", inference: { routeProvider: "inference", @@ -75,7 +91,8 @@ const startup = buildManagedStartupProfile({ observabilityEnabled: null, environment: {}, corporateCa: null, -}); +} satisfies ManagedStartupProfileBuilderInput; +const startup = buildManagedStartupProfile(startupInput); const entry: SandboxEntry = { name: "alpha", @@ -194,6 +211,83 @@ function mockSupportedLiveSource( raw.getSandboxConfig.mockResolvedValue(configuration(appliedRevision)); } +function braveProvider() { + const readCredential = vi.fn(() => { + throw new Error(readFailureCanary); + }); + const credentials = Object.defineProperty({}, "BRAVE_API_KEY", { + enumerable: true, + get: readCredential, + }); + return { + readCredential, + provider: { + metadata: { + id: "brave-id", + name: "alpha-brave-search", + workspace: "default", + resourceVersion: 9n, + }, + type: "brave", + profileWorkspace: "default", + credentials, + config: {}, + }, + }; +} + +function mockBraveLiveSource() { + const built = buildManagedStartupProfile({ + ...startupInput, + webSearch: { fetchEnabled: true, provider: "brave" }, + }); + mockSupportedLiveSource(3, 3, { + ...entry, + webSearchEnabled: true, + webSearchProvider: "brave", + workload: { + ...(entry.workload as Extract< + NonNullable, + { kind: "managed-image" } + >), + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + }, + }); + const search = braveProvider(); + raw.getProviderProfile.mockResolvedValue({ profile: managedBraveProfile() }); + raw.getProvider.mockImplementation(async ({ name }: { name: string }) => + name === "alpha-brave-search" ? { provider: search.provider } : provider(), + ); + raw.getSandbox.mockResolvedValue({ + sandbox: { + ...inventory().sandbox, + spec: { template: { image: imageRef }, providers: ["alpha-brave-search"] }, + }, + }); + return search; +} + +async function exportLiveSource() { + const writeStdout = vi.fn(async (_yaml: string) => {}); + const publish = vi.fn(); + const result = await runConfigExport( + { + sandboxName: "alpha", + documentName: parseNemoClawConfigDocumentName("alpha"), + target: { kind: "stdout" }, + }, + { + observe: (name) => observeStableExportSource(name, createLiveExportSnapshotReader()), + createDocumentUid: () => + parseNemoClawConfigDocumentUid("123e4567-e89b-42d3-a456-426614174001"), + writeStdout, + publish, + }, + ); + return { result, writeStdout, publish }; +} + function nativeNvidiaProvider() { return { ...provider().provider, type: "nvidia", profileWorkspace: "", config: {} }; } @@ -214,6 +308,138 @@ function mockNativeNvidiaSource() { } describe("live export snapshot reader", () => { + it("exports Brave through SDK metadata without reading its credential value (#10904)", async () => { + const search = mockBraveLiveSource(); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toEqual({ ok: true, completion: { kind: "stdout" } }); + const yaml = writeStdout.mock.calls[0]![0]; + const document = validateNemoClawConfig(YAML.parse(yaml)); + expect(document.spec.sandboxes[0]!.integrations?.webSearch).toEqual({ + provider: "brave", + agentRefs: ["primary"], + credential: { env: "BRAVE_API_KEY" }, + }); + expect(document.spec.inferenceProviders).toHaveLength(1); + expect(search.readCredential).not.toHaveBeenCalled(); + expect(yaml).not.toContain(readFailureCanary); + expect(raw.getProvider.mock.calls.map(([request]) => request.name)).toEqual([ + "nvidia-prod", + "alpha-brave-search", + "nvidia-prod", + "alpha-brave-search", + ]); + expect(raw.getProviderProfile).toHaveBeenCalledTimes(2); + expect(raw.getProviderProfile).toHaveBeenCalledWith( + { id: "brave", workspace: "default" }, + { signal: expect.any(AbortSignal) }, + ); + expect(publish).not.toHaveBeenCalled(); + }); + + it.each([ + { type: "generic" }, + { profileWorkspace: undefined }, + { profileWorkspace: "foreign" }, + { credentials: { OTHER_API_KEY: readFailureCanary } }, + { config: { BASE_URL: readFailureCanary } }, + ])("rejects unsupported Brave provider metadata without output %j (#10904)", async (change) => { + const search = mockBraveLiveSource(); + raw.getProvider.mockImplementation(async ({ name }: { name: string }) => + name === "alpha-brave-search" ? { provider: { ...search.provider, ...change } } : provider(), + ); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toMatchObject({ ok: false }); + expect(writeStdout).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + expect(search.readCredential).not.toHaveBeenCalled(); + expect(JSON.stringify(result)).not.toContain(readFailureCanary); + }); + + it("sanitizes a failed Brave metadata read before publication (#10904)", async () => { + mockBraveLiveSource(); + raw.getProvider + .mockResolvedValueOnce(provider()) + .mockRejectedValueOnce(new Error(readFailureCanary)); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toMatchObject({ ok: false }); + expect(JSON.stringify(result)).not.toContain(readFailureCanary); + expect(writeStdout).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + }); + + it.each([ + { source: "interceptor/foreign" }, + { source: "user", scope: "platform" }, + { resourceVersion: 0n }, + { endpoints: [] }, + { binaries: [] }, + { credentials: [] }, + ])("rejects a shadowed or changed Brave profile %# (#10904)", async (change) => { + const search = mockBraveLiveSource(); + raw.getProviderProfile.mockResolvedValue({ profile: { ...managedBraveProfile(), ...change } }); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toMatchObject({ ok: false }); + expect(writeStdout).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + expect(search.readCredential).not.toHaveBeenCalled(); + }); + + it("sanitizes a failed Brave profile read before publication (#10904)", async () => { + const search = mockBraveLiveSource(); + raw.getProviderProfile.mockRejectedValue(new Error(readFailureCanary)); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toMatchObject({ ok: false }); + expect(JSON.stringify(result)).not.toContain(readFailureCanary); + expect(writeStdout).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + expect(search.readCredential).not.toHaveBeenCalled(); + }); + + it("rejects a changing managed Brave profile revision without output (#10904)", async () => { + const search = mockBraveLiveSource(); + let revision = 10n; + raw.getProviderProfile.mockImplementation(async () => ({ + profile: { ...managedBraveProfile(), resourceVersion: revision++ }, + })); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toMatchObject({ + ok: false, + failure: { findings: [expect.objectContaining({ category: "unstable-source" })] }, + }); + expect(writeStdout).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + expect(search.readCredential).not.toHaveBeenCalled(); + }); + + it.each(["id", "resourceVersion"])( + "rejects changing Brave provider %s without output (#10904)", + async (field) => { + const search = mockBraveLiveSource(); + let revision = 10; + raw.getProvider.mockImplementation(async ({ name }: { name: string }) => + name === "alpha-brave-search" + ? { + provider: { + ...search.provider, + metadata: { + ...search.provider.metadata, + [field]: field === "id" ? `provider-${revision++}` : BigInt(revision++), + }, + }, + } + : provider(), + ); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toMatchObject({ + ok: false, + failure: { findings: [expect.objectContaining({ category: "unstable-source" })] }, + }); + expect(writeStdout).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + expect(search.readCredential).not.toHaveBeenCalled(); + }, + ); + it.each([ { stage: "registry", @@ -640,3 +866,304 @@ describe("live export snapshot reader", () => { expect(JSON.stringify(result)).not.toContain(canary); }); }); + +function mockManagedVllmSource( + environmentOverrides: NodeJS.ProcessEnv = {}, + webSearch: ManagedStartupProfileBuilderInput["webSearch"] = null, +) { + const catalog = loadServingCatalog(); + const provenance = servingProfileProvenance(catalog, EXPORTED_VLLM_PROFILE_ID); + const recipe = catalog.recipes.find(({ metadata }) => metadata.id === EXPORTED_VLLM_RECIPE_ID)!; + const model = recipe.spec.model.servedName!; + const runtimeImage = provenance.runtimeImage as ImmutableImageReference; + const inference = resolveManagedStartupInferenceRoute( + "openclaw", + "vllm-local", + model, + "openai-completions", + ); + const environment: NodeJS.ProcessEnv = {}; + // This is the actual onboarding projection of the fixed server's /v1/models response. + applyVllmRuntimeContextWindow({ data: [{ id: model, max_model_len: 65536 }] }, model, { + env: environment, + logger: { log: vi.fn(), warn: vi.fn() }, + }); + Object.assign(environment, environmentOverrides); + const built = buildManagedStartupProfile({ + agent: "openclaw", + inference: { + routeProvider: inference.providerKey, + upstreamProvider: "vllm-local", + model, + routedBaseUrl: inference.inferenceBaseUrl, + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: inference.primaryModelRef, + compatibility: inference.inferenceCompat ?? {}, + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + webSearch, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + corporateCa: null, + environment, + }); + const source: SandboxEntry = { + ...entry, + provider: "vllm-local", + model, + endpointUrl: "http://host.openshell.internal:18000/v1", + credentialEnv: null, + servingProfileProvenance: provenance, + webSearchEnabled: webSearch !== null, + webSearchProvider: webSearch?.provider ?? null, + workload: { + ...entry.workload!, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + } as SandboxEntry["workload"], + }; + const observed: ObservedManagedVllmRuntime = { + containerId: "a".repeat(64), + imageId: `sha256:${"b".repeat(64)}`, + networkId: "c".repeat(64), + startedAt: "2026-09-10T12:00:00Z", + serving: { + backend: "vllm", + catalogDigest: provenance.catalogDigest, + profile: { id: EXPORTED_VLLM_PROFILE_ID, digest: provenance.preset.digest }, + recipe: { id: EXPORTED_VLLM_RECIPE_ID, digest: provenance.recipe.digest }, + model: { ...provenance.model, servedName: model }, + runtime: { image: { ref: runtimeImage } }, + hostPort: 18000, + }, + }; + mockSupportedLiveSource(3, 3, source); + vi.mocked(observeManagedVllmForExport).mockReturnValue(observed); + vi.mocked(getSandboxEntryInference).mockReturnValue({ + kind: "configured", + provider: "vllm-local", + model, + }); + vi.mocked(getLiveGatewayInference).mockReturnValue({ + failure: null, + inference: { provider: "vllm-local", model }, + output: "", + status: 0, + }); + const liveSandbox = inventory(); + Object.assign(liveSandbox.sandbox.spec, { providers: ["vllm-local"] }); + raw.getSandbox.mockResolvedValue(liveSandbox); + const credentials = { NEMOCLAW_VLLM_LOCAL_TOKEN: readFailureCanary }; + raw.getProvider.mockResolvedValue({ + provider: { + metadata: { + id: "provider-id", + name: "vllm-local", + workspace: "default", + resourceVersion: 8n, + }, + type: "openai", + profileWorkspace: "default", + credentials, + config: { OPENAI_BASE_URL: source.endpointUrl }, + }, + }); + raw.getProviderProfile.mockResolvedValue({ + profile: { + id: "openai", + source: "user", + scope: "workspace", + resourceVersion: 4n, + credentials: [], + endpoints: [], + binaries: [], + inferenceCapable: true, + }, + }); + return { source, observed }; +} + +describe("managed vLLM export pipeline", () => { + it("exports the real fixed onboarding profile and reparses its managed provider", async () => { + const f = mockManagedVllmSource(); + const output = vi.fn(async (_value: string) => {}); + const publish = vi.fn(); + const result = await runConfigExport( + { + sandboxName: "alpha", + documentName: parseNemoClawConfigDocumentName("alpha"), + target: { kind: "stdout" }, + }, + { + observe: (name) => observeStableExportSource(name, createLiveExportSnapshotReader()), + createDocumentUid: () => + parseNemoClawConfigDocumentUid("123e4567-e89b-42d3-a456-426614174000"), + publish, + writeStdout: output, + }, + ); + expect(result).toEqual({ ok: true, completion: { kind: "stdout" } }); + const yaml = output.mock.calls[0]![0]; + const document = validateNemoClawConfig(YAML.parse(yaml)); + expect(document.spec.inferenceProviders).toEqual([ + { + name: "managed-vllm", + provider: "vllm-local", + api: "openai-completions", + serving: f.observed.serving, + }, + ]); + expect(document.spec.sandboxes[0]!.agents[0]!.inference.routes[0]!.overrides).toEqual({ + model: f.source.model, + contextWindow: 65536, + }); + expect(yaml).not.toContain(readFailureCanary); + expect(yaml).not.toContain("NEMOCLAW_VLLM_LOCAL_TOKEN"); + expect(yaml).not.toContain("host.openshell.internal"); + expect(publish).not.toHaveBeenCalled(); + }); + + it("exports managed vLLM and Brave with both qualified profile bindings", async () => { + const f = mockManagedVllmSource({}, { fetchEnabled: true, provider: "brave" }); + const search = braveProvider(); + const readManagedProvider = raw.getProvider.getMockImplementation()!; + const readManagedProfile = raw.getProviderProfile.getMockImplementation()!; + raw.getProvider.mockImplementation(async (request: { name: string }) => + request.name === "alpha-brave-search" + ? { provider: search.provider } + : readManagedProvider(request), + ); + raw.getProviderProfile.mockImplementation(async (request: { id: string }) => + request.id === "brave" ? { profile: managedBraveProfile() } : readManagedProfile(request), + ); + const liveSandbox = inventory(); + Object.assign(liveSandbox.sandbox.spec, { providers: ["vllm-local", "alpha-brave-search"] }); + raw.getSandbox.mockResolvedValue(liveSandbox); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toEqual({ ok: true, completion: { kind: "stdout" } }); + const document = validateNemoClawConfig(YAML.parse(writeStdout.mock.calls[0]![0])); + expect(document.spec.inferenceProviders[0]).toMatchObject({ serving: f.observed.serving }); + expect(document.spec.sandboxes[0]!.integrations?.webSearch).toEqual({ + provider: "brave", + agentRefs: ["primary"], + credential: { env: "BRAVE_API_KEY" }, + }); + expect(search.readCredential).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + }); + + it.each(["NEMOCLAW_MAX_TOKENS", "NEMOCLAW_AGENT_TIMEOUT"])( + "rejects unrepresented %s instead of losing it", + async (field) => { + mockManagedVllmSource({ [field]: "8192" }); + const result = await observeStableExportSource("alpha", createLiveExportSnapshotReader()); + expect(result).toMatchObject({ + ok: false, + findings: expect.arrayContaining([ + expect.objectContaining({ + category: "unsupported", + field: "source.workload.startupProfile", + }), + ]), + }); + }, + ); + + it("rejects a changed managed route and keeps publication unreachable", async () => { + const f = mockManagedVllmSource(); + vi.mocked(observeManagedVllmForExport).mockReturnValue({ + ...f.observed, + serving: { ...f.observed.serving, hostPort: 19000 }, + }); + const result = await observeStableExportSource("alpha", createLiveExportSnapshotReader()); + expect(result).toMatchObject({ + ok: false, + findings: expect.arrayContaining([ + expect.objectContaining({ field: "spec.inferenceProviders[].serving" }), + ]), + }); + }); + + it("detects managed container restart between complete snapshots", async () => { + const f = mockManagedVllmSource(); + let revision = 0; + vi.mocked(observeManagedVllmForExport).mockImplementation(() => ({ + ...f.observed, + startedAt: String(revision++), + })); + expect( + await observeStableExportSource("alpha", createLiveExportSnapshotReader()), + ).toMatchObject({ + ok: false, + attempts: 2, + findings: [expect.objectContaining({ category: "unstable-source" })], + }); + }); + + it("rejects a shadowed OpenAI profile with additional endpoint behavior", async () => { + mockManagedVllmSource(); + raw.getProviderProfile.mockResolvedValue({ + profile: { + id: "openai", + source: "user", + scope: "workspace", + resourceVersion: 4n, + credentials: [], + endpoints: [{ host: "unexpected.example", port: 443 }], + binaries: [], + inferenceCapable: true, + }, + }); + expect(await createLiveExportSnapshotReader().read("alpha")).toEqual({ + kind: "read-failed", + stage: "provider-metadata", + }); + }); + + it("detects resolved provider profile revision changes", async () => { + mockManagedVllmSource(); + let revision = 4n; + raw.getProviderProfile.mockImplementation(async () => ({ + profile: { + id: "openai", + source: "user", + scope: "workspace", + resourceVersion: revision++, + credentials: [], + endpoints: [], + binaries: [], + inferenceCapable: true, + }, + })); + expect( + await observeStableExportSource("alpha", createLiveExportSnapshotReader()), + ).toMatchObject({ + ok: false, + attempts: 2, + findings: [expect.objectContaining({ category: "unstable-source" })], + }); + }); + + it("contains runtime failures before provider metadata or publication", async () => { + mockManagedVllmSource(); + vi.mocked(observeManagedVllmForExport).mockImplementation(() => { + throw new Error(readFailureCanary); + }); + expect(await createLiveExportSnapshotReader().read("alpha")).toEqual({ + kind: "read-failed", + stage: "managed-serving", + }); + expect(raw.getProvider).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/adapters/config/live-export-source.ts b/src/lib/adapters/config/live-export-source.ts index 5f802357077..a52c07192ea 100644 --- a/src/lib/adapters/config/live-export-source.ts +++ b/src/lib/adapters/config/live-export-source.ts @@ -5,7 +5,7 @@ import os from "node:os"; import { isDeepStrictEqual } from "node:util"; import { isValidNemoClawPort } from "../../config/model"; -import { createProviders } from "../openshell/providers"; +import { createProviders, type Provider } from "../openshell/providers"; import { createSandboxes, type Sandbox } from "../openshell/sandboxes"; import { createSandboxConfig } from "../openshell/sandbox-config"; import { captureSanitizedResolvedOpenshell } from "../openshell/sanitized-capture"; @@ -17,12 +17,16 @@ import type { ExportSnapshotReader, ObservedExportGateway, ObservedExportInference, + ObservedExportWebSearchProvider, + ObservedManagedVllmRuntime, ObservedExportEndpointEvidence, ObservedExportRegistry, ObservedExportSandboxIdentity, RawExportSnapshot, } from "../../domain/config/export-evidence"; import { getLiveGatewayInference } from "../../inference/live"; +import { VLLM_LOCAL_CREDENTIAL_ENV } from "../../inference/serving/vllm-credential-contract"; +import { observeManagedVllmForExport } from "../../inference/serving/vllm-export-runtime"; import { normalizeInferenceSelection } from "../../inference/selection"; import { resolveGatewayName } from "../../onboard/gateway-binding/identity"; import { @@ -122,22 +126,52 @@ function providerContract(api: string | null | undefined) { return { type, configKey: "OPENAI_BASE_URL" } as const; } +function providerIdentity(provider: Provider, gatewayName: string, managed: boolean) { + return { + gatewayName, + workspace: provider.workspace, + name: provider.name, + id: provider.id, + resourceVersion: provider.resourceVersion, + ...(managed + ? { profileWorkspace: provider.profileWorkspace, managedProfile: provider.managedProfile } + : {}), + }; +} + +function expectedCredentialKeys(credentialEnv: string | null, managed: boolean): string[] { + if (managed) return [VLLM_LOCAL_CREDENTIAL_ENV]; + return credentialEnv === null ? [] : [credentialEnv]; +} + +function inferenceTopology( + entry: Readonly, + managed: boolean, +): ObservedExportInference["topology"] { + if (managed) return "managed"; + return entry.hostLocalInferenceReceipt || entry.hostLocalInferenceProvenance || entry.nimContainer + ? "local" + : "hosted"; +} + async function readProviderEvidence( normalized: ReturnType, routeProvider: string, gatewayName: string, signal: AbortSignal, + managedServing?: ObservedManagedVllmRuntime, ): Promise { const { type, configKey } = providerContract(normalized.preferredInferenceApi); const provider = await createProviders().get({ target: namedOpenShellGateway(gatewayName), workspace: "default", name: routeProvider, + ...(managedServing ? { profileContract: "openai" as const } : {}), configKeys: [configKey], signal, }); if (!provider) throw new Error("The live inference provider is missing."); - const credentialKeys = normalized.credentialEnv === null ? [] : [normalized.credentialEnv]; + const credentialKeys = expectedCredentialKeys(normalized.credentialEnv, !!managedServing); const builtin = provider.builtinInferenceEndpoint !== undefined; if ( type === null || @@ -149,13 +183,7 @@ async function readProviderEvidence( throw new Error("The live inference provider metadata does not match the registry."); } return { - provider: { - gatewayName, - workspace: provider.workspace, - name: provider.name, - id: provider.id, - resourceVersion: provider.resourceVersion, - }, + provider: providerIdentity(provider, gatewayName, !!managedServing), endpoint: provider.builtinInferenceEndpoint ?? provider.config[configKey] ?? "", source: builtin ? { kind: "builtin-profile", profileId: "nvidia" } @@ -167,6 +195,7 @@ async function inferenceFor( entry: Readonly, beforeProviderRead: () => void, signal: AbortSignal, + managedServing?: ObservedManagedVllmRuntime, ): Promise { const normalized = normalizeInferenceSelection(entry); const gateway = resolveGatewayBinding(entry); @@ -177,18 +206,47 @@ async function inferenceFor( live.provider, gateway.name, signal, + managedServing, ); return { - topology: - entry.hostLocalInferenceReceipt || entry.hostLocalInferenceProvenance || entry.nimContainer - ? "local" - : "hosted", + topology: inferenceTopology(entry, !!managedServing), provider: live.provider, model: live.model, api: normalized.preferredInferenceApi ?? "", endpoint: normalized.endpointUrl ?? "", endpointEvidence, credentialEnv: normalized.credentialEnv, + ...(managedServing ? { managedServing } : {}), + }; +} + +async function readWebSearchProvider( + entry: Readonly, + gatewayName: string, + signal: AbortSignal, +): Promise { + const provider = await createProviders().get({ + target: namedOpenShellGateway(gatewayName), + workspace: "default", + name: `${entry.name}-brave-search`, + configKeys: [], + profileContract: "brave", + signal, + }); + if (!provider) throw new Error("The live web-search provider is missing."); + return { + gatewayName, + workspace: provider.workspace, + name: provider.name, + id: provider.id, + resourceVersion: provider.resourceVersion, + type: provider.type, + ...(provider.profileWorkspace === undefined + ? {} + : { profileWorkspace: provider.profileWorkspace }), + ...(provider.managedProfile === undefined ? {} : { profile: provider.managedProfile }), + credentialKeys: provider.credentialKeys, + configKeys: provider.configKeys, }; } @@ -239,6 +297,11 @@ async function readSnapshot(sandboxName: string): Promise { if (!row) throw new Error("The live sandbox is missing."); stage = "sandbox-identity"; const sandbox = sandboxIdentity(row); + stage = "managed-serving"; + const managedServing = + entry.provider === "vllm-local" && entry.servingProfileProvenance + ? observeManagedVllmForExport(entry.servingProfileProvenance) + : undefined; stage = "inference-route"; const inference = await inferenceFor( entry, @@ -246,7 +309,13 @@ async function readSnapshot(sandboxName: string): Promise { stage = "provider-metadata"; }, signal, + managedServing, ); + let webSearchProvider: ObservedExportWebSearchProvider | undefined; + if (entry.webSearchEnabled === true && entry.webSearchProvider === "brave") { + stage = "web-search-provider"; + webSearchProvider = await readWebSearchProvider(entry, gateway.name, signal); + } stage = "effective-policy"; const { configuration, ...policy } = await effectivePolicy(gateway, row, signal); return { @@ -256,6 +325,7 @@ async function readSnapshot(sandboxName: string): Promise { gateway, sandbox, inference, + ...(webSearchProvider === undefined ? {} : { webSearchProvider }), policy, configuration, }; diff --git a/src/lib/adapters/openshell/README.md b/src/lib/adapters/openshell/README.md index 76568e54a2b..bd07de8a76f 100644 --- a/src/lib/adapters/openshell/README.md +++ b/src/lib/adapters/openshell/README.md @@ -8,7 +8,7 @@ issue #10938 and PR #11065. They do not complete the capability migrations in th | Read | Owner | Transport and reason | | --- | --- | --- | -| Provider endpoint and identity | `providers.ts` | SDK `raw.getProvider`, plus `raw.getProviderProfile` for native NVIDIA inference without overrides; the pinned SDK has no curated gateway-provider read. Reuses the metadata fields from `provider-adapter.ts` (#9806, #9825). | +| Provider endpoint and identity | `providers.ts` | SDK `raw.getProvider`, plus `raw.getProviderProfile` for native NVIDIA inference without overrides and requested managed Brave/OpenAI contracts; the pinned SDK has no curated gateway-provider read. Reuses the metadata fields from `provider-adapter.ts` (#9806, #9825). | | Sandbox identity, image, and attachments | `sandboxes.ts` | SDK `raw.getSandbox`; curated `sandbox.get` omits workspace, image, and active policy version. | | Configuration identity and effective policy | `sandbox-config.ts` | SDK `raw.getSandboxConfig` by verified ID returns both in one response; curated `sandbox.getConfig` does a new name lookup and omits workspace. Policy reads for other consumers remain with #9805 and #9826. | | Inference route | `inference/live.ts` | Retains the CLI read with an explicit gateway. The separate generated inference client remains with #9809 and #9828. | @@ -38,6 +38,21 @@ The pinned OpenShell native resolver uses `/v1` on that host. Export records the as the endpoint evidence. Custom profiles, profile scope changes, and provider config overrides cannot use this derivation. +Consumers can request `profileContract: "brave"` or `"openai"` to qualify a managed profile. +The reader resolves `raw.getProviderProfile` at the provider's `profileWorkspace` through the +same gateway. Normal onboarding imports the checked-in profile in the `default` workspace. +User profiles must have a nonzero revision and a scope matching their binding; builtin profiles +must have global binding, empty scope, and revision zero. Matching the profile name is insufficient. + +Qualification requires the checked-in credential declaration, endpoint rules, binary allowlist, +and inference capability. Brave permits its single header credential and search endpoint; +OpenAI requires the endpointless inference contract. Credential refresh, token grants, discovery, +changed rewriting rules, and unknown protobuf fields in the profile's semantic messages fail. +Provider credential values and handles remain opaque. The reader returns profile identity, +source, scope, revision, and binding for inclusion in complete export observations. A changed +binding or profile revision therefore prevents publication until observations agree. Hosted +consumers that do not request this qualification retain their existing endpoint semantics. + Provider reads retain the complete config-key inventory so export can reject unsupported configuration. Sandbox reads omit environment values. Configuration reads return revision metadata and a credential-free effective policy document, without settings values. The policy conversion follows the reviewed OpenShell release: filesystem defaults, compact ports, diff --git a/src/lib/adapters/openshell/providers.test.ts b/src/lib/adapters/openshell/providers.test.ts index 3b9e5f4651a..9cf8da371a0 100644 --- a/src/lib/adapters/openshell/providers.test.ts +++ b/src/lib/adapters/openshell/providers.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; +import { managedBraveProfile } from "../../../../test/fixtures/openshell-provider-profile"; import { createProviders } from "./providers"; import { createSandboxes } from "./sandboxes"; import { createSandboxConfig } from "./sandbox-config"; @@ -114,6 +115,217 @@ describe("OpenShell provider evidence", () => { expect(result?.config.OPENAI_BASE_URL).toBe("https://api.example/v1"); }); + it("never evaluates credential or unrequested configuration values (#10904)", async () => { + const { connect, raw } = fixture(); + const readSecret = vi.fn(() => { + throw new Error(canary); + }); + const opaque = () => Object.defineProperty({}, "SECRET", { enumerable: true, get: readSecret }); + raw.getProvider.mockResolvedValue({ + provider: { + ...provider().provider, + credentials: opaque(), + credentialHandles: opaque(), + config: Object.assign(opaque(), { OPENAI_BASE_URL: "https://api.example/v1" }), + }, + }); + const result = await createProviders(connect).get(request()); + expect(result?.credentialKeys).toEqual(["SECRET"]); + expect(result?.config).toEqual({ OPENAI_BASE_URL: "https://api.example/v1" }); + expect(result?.configKeys).toEqual(["OPENAI_BASE_URL", "SECRET"]); + expect(readSecret).not.toHaveBeenCalled(); + expect(JSON.stringify(result)).not.toContain(canary); + }); + + it.each([ + { profileWorkspace: "default", source: "user", scope: "workspace", resourceVersion: 4n }, + { profileWorkspace: "", source: "user", scope: "platform", resourceVersion: 7n }, + { profileWorkspace: "", source: "builtin", scope: "", resourceVersion: 0n }, + ])( + "qualifies exact managed Brave profile semantics at its binding %# (#10904)", + async ({ profileWorkspace, ...identity }) => { + const { connect, raw } = fixture(); + const readSecret = vi.fn(() => { + throw new Error(canary); + }); + raw.getProvider.mockResolvedValue({ + provider: { + ...provider().provider, + type: "brave", + profileWorkspace, + credentials: Object.defineProperty({}, "BRAVE_API_KEY", { + enumerable: true, + get: readSecret, + }), + credentialHandles: {}, + config: {}, + }, + }); + raw.getProviderProfile.mockResolvedValue({ + profile: { ...managedBraveProfile(), ...identity }, + }); + const input = { ...request(), configKeys: [], profileContract: "brave" as const }; + const result = await createProviders(connect).get(input); + expect(result).toMatchObject({ + profileWorkspace, + managedProfile: { + id: "brave", + ...identity, + resourceVersion: String(identity.resourceVersion), + }, + }); + expect(raw.getProviderProfile).toHaveBeenCalledExactlyOnceWith( + { id: "brave", workspace: profileWorkspace }, + { signal: input.signal }, + ); + expect(readSecret).not.toHaveBeenCalled(); + expect(Object.isFrozen(result?.managedProfile)).toBe(true); + }, + ); + + it.each([ + { label: "wrong identity", change: { id: "other" } }, + { label: "unknown profile semantics", change: { $unknown: [{}] } }, + { + label: "unknown binary semantics", + change: { + binaries: managedBraveProfile().binaries.map((binary) => ({ ...binary, $unknown: [{}] })), + }, + }, + { label: "unknown source", change: { source: "interceptor/foreign" } }, + { label: "platform shadow", change: { scope: "platform" } }, + { label: "builtin shadow", change: { source: "builtin", scope: "", resourceVersion: 0n } }, + { label: "missing revision", change: { resourceVersion: undefined } }, + { label: "zero custom revision", change: { resourceVersion: 0n } }, + { label: "inference enabled", change: { inferenceCapable: true } }, + { label: "missing credentials", change: { credentials: [] } }, + { label: "missing binaries", change: { binaries: [] } }, + { label: "discovery override", change: { discovery: {} } }, + ])("rejects managed Brave profiles with $label (#10904)", async ({ change }) => { + const { connect, raw } = fixture(); + raw.getProvider.mockResolvedValue({ + provider: { ...provider().provider, type: "brave", profileWorkspace: "default" }, + }); + raw.getProviderProfile.mockResolvedValue({ profile: { ...managedBraveProfile(), ...change } }); + await expect( + createProviders(connect).get({ ...request(), profileContract: "brave" }), + ).rejects.toMatchObject({ kind: "schema" }); + }); + + it.each([ + { $unknown: [{}] }, + { host: "foreign.example" }, + { port: 80 }, + { ports: [80] }, + { protocol: "" }, + { access: "full" }, + { enforcement: "audit" }, + { allowedIps: ["10.0.0.1"] }, + { requestBodyCredentialRewrite: true }, + { credentialSigning: "sigv4" }, + { signingService: "bedrock" }, + { signingRegion: "us-east-1" }, + { jsonRpcMaxBodyBytes: 100 }, + { mcp: {} }, + { credentialBinding: { provider: "foreign", credential: "OTHER_KEY" } }, + { websocketCredentialRewrite: true }, + { allowEncodedSlash: true }, + { path: "/other" }, + ])("rejects managed Brave endpoint drift %j (#10904)", async (change) => { + const { connect, raw } = fixture(); + const profile = managedBraveProfile(); + raw.getProvider.mockResolvedValue({ + provider: { ...provider().provider, type: "brave", profileWorkspace: "default" }, + }); + raw.getProviderProfile.mockResolvedValue({ + profile: { ...profile, endpoints: [{ ...profile.endpoints[0], ...change }] }, + }); + await expect( + createProviders(connect).get({ ...request(), profileContract: "brave" }), + ).rejects.toMatchObject({ kind: "schema" }); + }); + + it.each([ + { $unknown: [{}] }, + { envVars: ["OTHER_KEY"] }, + { authStyle: "bearer" }, + { headerName: "Authorization" }, + { queryParam: "token" }, + { pathTemplate: "/token/{token}" }, + { refresh: {} }, + { tokenGrant: {} }, + ])("rejects managed Brave credential declaration drift %j (#10904)", async (change) => { + const { connect, raw } = fixture(); + const profile = managedBraveProfile(); + raw.getProvider.mockResolvedValue({ + provider: { ...provider().provider, type: "brave", profileWorkspace: "default" }, + }); + raw.getProviderProfile.mockResolvedValue({ + profile: { ...profile, credentials: [{ ...profile.credentials[0], ...change }] }, + }); + await expect( + createProviders(connect).get({ ...request(), profileContract: "brave" }), + ).rejects.toMatchObject({ kind: "schema" }); + }); + + it("qualifies the endpointless managed OpenAI profile only when requested (#10904)", async () => { + const { connect, raw } = fixture(); + raw.getProvider.mockResolvedValue({ + provider: { ...provider().provider, profileWorkspace: "default" }, + }); + raw.getProviderProfile.mockResolvedValue({ + profile: { + id: "openai", + source: "user", + scope: "workspace", + resourceVersion: 4n, + inferenceCapable: true, + credentials: [], + endpoints: [], + binaries: [], + }, + }); + await createProviders(connect).get(request()); + expect(raw.getProviderProfile).not.toHaveBeenCalled(); + const result = await createProviders(connect).get({ ...request(), profileContract: "openai" }); + expect(result?.managedProfile).toEqual({ + id: "openai", + source: "user", + scope: "workspace", + resourceVersion: "4", + }); + }); + + it.each([ + { $unknown: [{}] }, + { credentials: [{}] }, + { endpoints: [{ host: "foreign.example", port: 443 }] }, + { binaries: [{ path: "/usr/bin/curl" }] }, + { inferenceCapable: false }, + { discovery: {} }, + ])("rejects managed OpenAI profile drift %j (#10904)", async (change) => { + const { connect, raw } = fixture(); + raw.getProvider.mockResolvedValue({ + provider: { ...provider().provider, profileWorkspace: "default" }, + }); + raw.getProviderProfile.mockResolvedValue({ + profile: { + id: "openai", + source: "user", + scope: "workspace", + resourceVersion: 4n, + inferenceCapable: true, + credentials: [], + endpoints: [], + binaries: [], + ...change, + }, + }); + await expect( + createProviders(connect).get({ ...request(), profileContract: "openai" }), + ).rejects.toMatchObject({ kind: "schema" }); + }); + it("verifies the native NVIDIA endpoint through the named gateway profile", async () => { const { connect, raw } = nativeNvidiaFixture(); const input = request(); @@ -257,6 +469,9 @@ describe("OpenShell provider evidence", () => { it.each([ { credentials: [] }, + { credentials: new Date() }, + { credentialHandles: [] }, + { credentialHandles: new Map() }, { config: null }, { credentialHandles: [] }, { config: { OPENAI_BASE_URL: { secret: canary } } }, diff --git a/src/lib/adapters/openshell/providers.ts b/src/lib/adapters/openshell/providers.ts index 9d5d9763a2d..e1b33638472 100644 --- a/src/lib/adapters/openshell/providers.ts +++ b/src/lib/adapters/openshell/providers.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { isDeepStrictEqual } from "node:util"; import { connectOpenShellReader, isNotFound, @@ -12,9 +13,15 @@ import { type ConnectOpenShellReader, type ReadRequest, type OpenShellReadClient, + OpenShellReadError, } from "./sdk-read"; -import { BuiltinNvidiaProfileResponseSchema, ProviderResponseSchema } from "./sdk-read-schema"; +import { + ManagedBraveProfileResponseSchema, + ManagedOpenAiProfileResponseSchema, + BuiltinNvidiaProfileResponseSchema, + ProviderResponseSchema, +} from "./sdk-read-schema"; import { BUILD_ENDPOINT_URL } from "../../inference/provider-models"; @@ -27,11 +34,23 @@ export type Provider = Readonly< resourceVersion: string; config: Readonly>; builtinInferenceEndpoint?: string; + profileWorkspace?: string; + managedProfile?: Readonly<{ + id: "brave" | "openai"; + source: "builtin" | "user"; + scope: "" | "platform" | "workspace"; + resourceVersion: string; + }>; } >; export interface Providers { get( - request: ReadRequest & Readonly<{ name: string; configKeys: readonly string[] }>, + request: ReadRequest & + Readonly<{ + name: string; + configKeys: readonly string[]; + profileContract?: "brave" | "openai"; + }>, ): Promise; } @@ -50,6 +69,77 @@ async function readBuiltinNvidiaEndpoint( return BUILD_ENDPOINT_URL; } +async function readManagedProfile( + client: OpenShellReadClient, + request: ReadRequest, + profileId: "brave" | "openai", + providerType: string, + profileWorkspace: string | undefined, +): Promise> { + if ( + providerType !== profileId || + (profileWorkspace !== "" && profileWorkspace !== request.workspace) + ) + throw new OpenShellReadError("schema"); + request.signal.throwIfAborted(); + const { profile } = readValue( + profileId === "brave" ? ManagedBraveProfileResponseSchema : ManagedOpenAiProfileResponseSchema, + await client.raw.getProviderProfile( + // Resolve the actual profile binding; the default-workspace import is managed state. + { id: profileId, workspace: profileWorkspace }, + { signal: request.signal }, + ), + ); + const builtin = profile.source === "builtin"; + const customScope = profileWorkspace === "" ? "platform" : "workspace"; + const expectedScope = builtin ? "" : customScope; + if ( + !isDeepStrictEqual( + [profile.scope, profileWorkspace, BigInt(profile.resourceVersion) === 0n], + [expectedScope, builtin ? "" : profileWorkspace, builtin], + ) + ) + throw new OpenShellReadError("schema"); + return { + id: profile.id, + source: profile.source, + scope: profile.scope, + resourceVersion: String(profile.resourceVersion), + }; +} + +async function readProfileEvidence( + client: OpenShellReadClient, + request: Parameters[0], + provider: Readonly<{ type: string; profileWorkspace?: string; config: Record }>, +): Promise> { + let builtinInferenceEndpoint: string | undefined; + if ( + provider.type === "nvidia" && + provider.profileWorkspace === "" && + Object.keys(provider.config).length === 0 + ) { + builtinInferenceEndpoint = await readBuiltinNvidiaEndpoint(client, request); + } + const managedProfile = + request.profileContract === undefined + ? undefined + : await readManagedProfile( + client, + request, + request.profileContract, + provider.type, + provider.profileWorkspace, + ); + return { + ...(builtinInferenceEndpoint === undefined ? {} : { builtinInferenceEndpoint }), + ...(provider.profileWorkspace === undefined + ? {} + : { profileWorkspace: provider.profileWorkspace }), + ...(managedProfile === undefined ? {} : { managedProfile }), + }; +} + export function createProviders( connect: ConnectOpenShellReader = connectOpenShellReader, ): Providers { @@ -74,17 +164,10 @@ export function createProviders( const { provider } = readValue(ProviderResponseSchema, response); const { config } = provider; const identity = metadata(provider.metadata, name, request.workspace); - let builtinInferenceEndpoint: string | undefined; - if ( - provider.type === "nvidia" && - provider.profileWorkspace === "" && - Object.keys(config).length === 0 - ) { - builtinInferenceEndpoint = await readBuiltinNvidiaEndpoint(client, request); - } + const profileEvidence = await readProfileEvidence(client, request, provider); return owned({ ...identity, - ...(builtinInferenceEndpoint === undefined ? {} : { builtinInferenceEndpoint }), + ...profileEvidence, type: provider.type, credentialKeys: [ ...new Set([ diff --git a/src/lib/adapters/openshell/sdk-read-schema.ts b/src/lib/adapters/openshell/sdk-read-schema.ts index cc06955afce..ef2e8519514 100644 --- a/src/lib/adapters/openshell/sdk-read-schema.ts +++ b/src/lib/adapters/openshell/sdk-read-schema.ts @@ -24,7 +24,13 @@ export const MetadataSchema = Type.Object({ }); // Validate only consumed fields. Credential values and unrequested config remain opaque. -const OpaqueMapSchema = Type.Record(Type.String(), Type.Unknown()); +const OpaqueMapSchema = Type.Unsafe>( + Type.Refine(Type.Unknown(), (value) => { + if (typeof value !== "object" || value === null) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; + }), +); export const ProviderResponseSchema = Type.Object({ provider: Type.Object({ metadata: MetadataSchema, @@ -49,6 +55,81 @@ export const BuiltinNvidiaProfileResponseSchema = Type.Object({ ]), }), }); +const NoUnknownProfileFields = { $unknown: Type.Optional(Type.Tuple([])) }; +const ManagedProfileIdentity = { + ...NoUnknownProfileFields, + source: Type.Union([Type.Literal("builtin"), Type.Literal("user")]), + scope: Type.Union([Type.Literal(""), Type.Literal("platform"), Type.Literal("workspace")]), + resourceVersion: VersionSchema, + discovery: Type.Optional(Type.Undefined()), +}; +// Match the current checked-in profiles, including credential rewriting and egress. +export const ManagedBraveProfileResponseSchema = Type.Object({ + profile: Type.Object({ + ...ManagedProfileIdentity, + id: Type.Literal("brave"), + inferenceCapable: Type.Literal(false), + endpoints: Type.Tuple([ + Type.Object({ + ...NoUnknownProfileFields, + host: Type.Literal("api.search.brave.com"), + port: Type.Literal(443), + ports: Type.Union([Type.Tuple([]), Type.Tuple([Type.Literal(443)])]), + protocol: Type.Literal("rest"), + tls: Type.Literal(""), + enforcement: Type.Literal("enforce"), + access: Type.Literal("read-write"), + rules: Type.Tuple([]), + allowedIps: Type.Tuple([]), + denyRules: Type.Tuple([]), + allowEncodedSlash: Type.Literal(false), + persistedQueries: Type.Literal(""), + graphqlPersistedQueries: Type.Object({}, { additionalProperties: false }), + graphqlMaxBodyBytes: Type.Literal(0), + path: Type.Literal(""), + websocketCredentialRewrite: Type.Literal(false), + requestBodyCredentialRewrite: Type.Literal(false), + advisorProposed: Type.Literal(false), + credentialSigning: Type.Literal(""), + signingService: Type.Literal(""), + signingRegion: Type.Literal(""), + jsonRpcMaxBodyBytes: Type.Literal(0), + mcp: Type.Optional(Type.Undefined()), + credentialBinding: Type.Optional(Type.Undefined()), + }), + ]), + credentials: Type.Tuple([ + Type.Object({ + ...NoUnknownProfileFields, + name: Type.Literal("api_key"), + envVars: Type.Tuple([Type.Literal("BRAVE_API_KEY")]), + required: Type.Literal(true), + authStyle: Type.Literal("header"), + headerName: Type.Literal("x-subscription-token"), + queryParam: Type.Literal(""), + pathTemplate: Type.Literal(""), + refresh: Type.Optional(Type.Undefined()), + tokenGrant: Type.Optional(Type.Undefined()), + }), + ]), + binaries: Type.Tuple([ + Type.Object({ ...NoUnknownProfileFields, path: Type.Literal("/usr/local/bin/node") }), + Type.Object({ ...NoUnknownProfileFields, path: Type.Literal("/usr/bin/node") }), + Type.Object({ ...NoUnknownProfileFields, path: Type.Literal("/usr/local/bin/curl") }), + Type.Object({ ...NoUnknownProfileFields, path: Type.Literal("/usr/bin/curl") }), + ]), + }), +}); +export const ManagedOpenAiProfileResponseSchema = Type.Object({ + profile: Type.Object({ + ...ManagedProfileIdentity, + id: Type.Literal("openai"), + inferenceCapable: Type.Literal(true), + credentials: Type.Tuple([]), + endpoints: Type.Tuple([]), + binaries: Type.Tuple([]), + }), +}); export const SandboxResponseSchema = Type.Object({ sandbox: Type.Object({ metadata: MetadataSchema, diff --git a/src/lib/config/config.test.ts b/src/lib/config/config.test.ts index 01216183530..e58cf3d5f1c 100644 --- a/src/lib/config/config.test.ts +++ b/src/lib/config/config.test.ts @@ -5,6 +5,8 @@ import YAML from "yaml"; import { describe, expect, it } from "vitest"; import { renderCanonicalNemoClawConfig, validateNemoClawConfig } from "./index"; import { + EXPORTED_VLLM_PROFILE_ID, + EXPORTED_VLLM_RECIPE_ID, isCredentialEnvironmentReferenceName, isImmutableImageReference, isValidNemoClawBoundedText, @@ -83,6 +85,67 @@ describe("NemoClawConfig v1", () => { expect(validateNemoClawConfig(config())).toEqual(config()); }); + it("round-trips Brave search bound to the primary agent (#10904)", () => { + const value = config(); + const integration = { + webSearch: { + provider: "brave", + agentRefs: ["primary"], + credential: { env: "BRAVE_API_KEY" }, + }, + }; + Object.assign(value.spec.sandboxes[0]!, { integrations: integration }); + const rendered = renderInput(value); + expect(validateNemoClawConfig(YAML.parse(rendered.yaml))).toEqual(value); + }); + + it.each([ + { provider: "tavily" }, + { agentRefs: [] }, + { agentRefs: ["primary", "primary"] }, + { agentRefs: ["other"] }, + { credential: { env: "NEMOCLAW_PROVIDER_KEY" } }, + { credential: { env: "TAVILY_API_KEY" } }, + { credential: { value: "secret-canary" } }, + { unexpected: true }, + ])("rejects unsupported Brave configuration %j (#10904)", (change) => { + const value = config(); + Object.assign(value.spec.sandboxes[0]!, { + integrations: { + webSearch: { + provider: "brave", + agentRefs: ["primary"], + credential: { env: "BRAVE_API_KEY" }, + ...change, + }, + }, + }); + expect(() => validateNemoClawConfig(value)).toThrow(); + try { + validateNemoClawConfig(value); + } catch (error) { + expect(String(error)).not.toContain("secret-canary"); + } + }); + + it.each([ + { label: "missing primary", change: { name: "other" } }, + { label: "wrong type", change: { type: "hermes" } }, + ])("rejects a Brave binding with $label (#10904)", ({ change }) => { + const value = config(); + Object.assign(value.spec.sandboxes[0]!, { + integrations: { + webSearch: { + provider: "brave", + agentRefs: ["primary"], + credential: { env: "BRAVE_API_KEY" }, + }, + }, + }); + Object.assign(value.spec.sandboxes[0]!.agents[0]!, change); + expect(() => validateNemoClawConfig(value)).toThrow(); + }); + it.each([ { host: "user:secret-canary@proxy.internal", port: 3129 }, { host: "http://proxy.internal", port: 3129 }, @@ -419,3 +482,111 @@ describe("NemoClawConfig v1", () => { expect(first.documentDigest).toMatch(/^sha256:[0-9a-f]{64}$/u); }); }); + +function managedServingConfig() { + const value = config(); + const provider = { + name: "managed-vllm", + provider: "vllm-local", + api: "openai-completions", + serving: { + backend: "vllm", + catalogDigest: `sha256:${"b".repeat(64)}`, + profile: { id: EXPORTED_VLLM_PROFILE_ID, digest: `sha256:${"c".repeat(64)}` }, + recipe: { id: EXPORTED_VLLM_RECIPE_ID, digest: `sha256:${"d".repeat(64)}` }, + model: { id: "nvidia/model", revision: "e".repeat(40), servedName: "managed-model" }, + runtime: { image: { ref: `nvcr.io/nvidia/vllm@sha256:${"f".repeat(64)}` } }, + hostPort: 18000, + }, + }; + Object.assign(value.spec, { inferenceProviders: [provider] }); + const route = { + name: "primary", + providerRef: "managed-vllm", + overrides: { model: "managed-model", contextWindow: 65536 }, + }; + value.spec.sandboxes[0]!.agents[0]!.inference.routes = [route]; + return { value, provider, route }; +} + +describe("fixed managed serving public contract", () => { + it("round trips an immutable catalog reference and nondefault published port", () => { + const { value } = managedServingConfig(); + expect(validateNemoClawConfig(YAML.parse(renderInput(value).yaml))).toEqual(value); + }); + + it("rejects vllm-local without its managed serving contract", () => { + const { value, provider } = managedServingConfig(); + Reflect.deleteProperty(provider, "serving"); + Object.assign(provider, { endpoint: "https://127.0.0.1:18000/v1" }); + expect(() => validateNemoClawConfig(value)).toThrow(); + }); + + it.each([ + [ + "transport credential", + (f: ReturnType) => + Object.assign(f.provider, { credential: { env: "NEMOCLAW_VLLM_LOCAL_TOKEN" } }), + ], + [ + "arbitrary endpoint", + (f: ReturnType) => + Object.assign(f.provider, { endpoint: "http://127.0.0.1:18000/v1" }), + ], + [ + "arbitrary arguments", + (f: ReturnType) => + Object.assign(f.provider.serving, { arguments: ["--trust-remote-code"] }), + ], + [ + "other recipe", + (f: ReturnType) => { + Object.assign(f.provider.serving.recipe, { id: "other" }); + }, + ], + [ + "mutable image", + (f: ReturnType) => { + f.provider.serving.runtime.image.ref = "vllm:latest"; + }, + ], + [ + "unknown backend", + (f: ReturnType) => { + f.provider.serving.backend = "ollama"; + }, + ], + [ + "unknown runtime property", + (f: ReturnType) => + Object.assign(f.provider.serving.runtime, { env: { SECRET: "private" } }), + ], + [ + "missing context", + (f: ReturnType) => + Reflect.deleteProperty(f.route.overrides, "contextWindow"), + ], + [ + "different context", + (f: ReturnType) => { + f.route.overrides.contextWindow = 32768; + }, + ], + [ + "different model", + (f: ReturnType) => { + f.route.overrides.model = "other"; + }, + ], + [ + "different runtime", + (f: ReturnType) => { + f.value.spec.sandboxes[0]!.runtime.provider = "apple-container"; + }, + ], + ])("rejects %s instead of accepting an incomplete managed intent", (_name, change) => { + const f = managedServingConfig(); + change(f); + expect(() => validateNemoClawConfig(f.value)).toThrow(); + }); +}); diff --git a/src/lib/config/model.ts b/src/lib/config/model.ts index 6d55702e1a4..1607cf53c93 100644 --- a/src/lib/config/model.ts +++ b/src/lib/config/model.ts @@ -83,6 +83,10 @@ export const BoundedTextSchema = Type.String({ maxLength: BOUNDED_TEXT_MAX_LENGTH, pattern: BOUNDED_TEXT_PATTERN, }); +const HostedInferenceProviderNameSchema = Type.Unsafe({ + ...BoundedTextSchema, + not: { const: "vllm-local" }, +}); export const RuntimeProviderSchema = Type.String({ pattern: RUNTIME_PROVIDER_PATTERN }); export const ImmutableImageReferenceSchema = Type.Unsafe({ type: "string", @@ -224,10 +228,10 @@ const NemoClawGatewayConfigSchema = Type.Object( { additionalProperties: false }, ); -const NemoClawInferenceProviderConfigSchema = Type.Object( +const NemoClawHostedInferenceProviderConfigSchema = Type.Object( { name: LocalResourceNameSchema, - provider: BoundedTextSchema, + provider: HostedInferenceProviderNameSchema, api: NemoClawInferenceApiSchema, endpoint: InferenceEndpointSchema, credential: Type.Optional(CredentialEnvironmentReferenceSchema), @@ -235,8 +239,69 @@ const NemoClawInferenceProviderConfigSchema = Type.Object( { additionalProperties: false }, ); +export const EXPORTED_VLLM_PROFILE_ID = + "vllm.linux-amd64-nvidia.single.nemotron-3.5-lightning-30b-a3b-nvfp4" as const; +export const EXPORTED_VLLM_CONTEXT_WINDOW = 65_536; +export const EXPORTED_VLLM_RECIPE_ID = + "vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.linux-amd64-single.v1" as const; + +const ServingDigestSchema = Type.String({ pattern: "^sha256:[a-f0-9]{64}$" }); + +/** The first managed-serving branch describes one fixed, catalog-owned deployment. */ +export const NemoClawManagedVllmServingSchema = Type.Object( + { + backend: Type.Literal("vllm"), + catalogDigest: ServingDigestSchema, + profile: Type.Object( + { id: Type.Literal(EXPORTED_VLLM_PROFILE_ID), digest: ServingDigestSchema }, + { additionalProperties: false }, + ), + recipe: Type.Object( + { id: Type.Literal(EXPORTED_VLLM_RECIPE_ID), digest: ServingDigestSchema }, + { additionalProperties: false }, + ), + model: Type.Object( + { + id: BoundedTextSchema, + revision: Type.String({ pattern: "^[a-f0-9]{40}$" }), + servedName: BoundedTextSchema, + }, + { additionalProperties: false }, + ), + runtime: Type.Object( + { + image: Type.Object({ ref: ImmutableImageReferenceSchema }, { additionalProperties: false }), + }, + { additionalProperties: false }, + ), + hostPort: Type.Integer({ minimum: 1024, maximum: 65_535 }), + }, + { additionalProperties: false }, +); +export type NemoClawManagedVllmServing = TypeBoxModule.Type.Static< + typeof NemoClawManagedVllmServingSchema +>; + +const NemoClawManagedInferenceProviderConfigSchema = Type.Object( + { + name: LocalResourceNameSchema, + provider: Type.Literal("vllm-local"), + api: Type.Literal("openai-completions"), + serving: NemoClawManagedVllmServingSchema, + }, + { additionalProperties: false }, +); + +const NemoClawInferenceProviderConfigSchema = Type.Union([ + NemoClawHostedInferenceProviderConfigSchema, + NemoClawManagedInferenceProviderConfigSchema, +]); + const NemoClawRouteOverridesSchema = Type.Object( - { model: BoundedTextSchema }, + { + model: BoundedTextSchema, + contextWindow: Type.Optional(Type.Integer({ minimum: 1, maximum: 4_194_304 })), + }, { additionalProperties: false }, ); @@ -280,6 +345,15 @@ const NemoClawExplicitPolicySchema = Type.Unsafe>({ $ref: NEMOCLAW_SANDBOX_POLICY_SCHEMA_ID, }); +export const NemoClawBraveSearchConfigSchema = Type.Object( + { + provider: Type.Literal("brave"), + agentRefs: Type.Array(Type.Literal("primary"), { minItems: 1, maxItems: 1 }), + credential: CredentialEnvironmentReferenceSchema, + }, + { additionalProperties: false }, +); + const NemoClawSandboxConfigSchema = Type.Object( { name: SandboxNameSchema, @@ -295,6 +369,9 @@ const NemoClawSandboxConfigSchema = Type.Object( { additionalProperties: false }, ), agents: Type.Array(NemoClawAgentConfigSchema, { minItems: 1 }), + integrations: Type.Optional( + Type.Object({ webSearch: NemoClawBraveSearchConfigSchema }, { additionalProperties: false }), + ), }, { additionalProperties: false }, ); diff --git a/src/lib/config/schema.ts b/src/lib/config/schema.ts index f991d289907..6818c214ee2 100644 --- a/src/lib/config/schema.ts +++ b/src/lib/config/schema.ts @@ -15,8 +15,10 @@ import { cloneAndDeepFreeze } from "../core/immutable"; import { isSandboxPolicyCredentialFree } from "../policy/sandbox-policy-validation"; import { isCredentialEnvironmentReferenceName, + EXPORTED_VLLM_CONTEXT_WINDOW, NemoClawConfigSchema, type NemoClawConfig, + type NemoClawInferenceProviderConfig, type NemoClawSandboxConfig, type ValidatedNemoClawConfig, } from "./model"; @@ -93,9 +95,56 @@ function sandboxProblems( ); } } + problems.push(...webSearchProblems(sandbox, sandboxIndex)); return problems; } +function webSearchProblems(sandbox: NemoClawSandboxConfig, sandboxIndex: number): string[] { + const problems: string[] = []; + const search = sandbox.integrations?.webSearch; + if (search) { + const location = `/spec/sandboxes/${sandboxIndex}/integrations/webSearch`; + if ( + !isCredentialEnvironmentReferenceName(search.credential.env) || + search.credential.env !== "BRAVE_API_KEY" + ) { + problems.push(`${location}/credential/env must reference the Brave credential`); + } + if ( + !search.agentRefs.every((name) => + sandbox.agents.some((agent) => agent.name === name && agent.type === "openclaw"), + ) + ) { + problems.push(`${location}/agentRefs must reference an OpenClaw agent in this sandbox`); + } + } + return problems; +} + +function managedProviderProblems( + config: NemoClawConfig, + provider: Extract, + providerIndex: number, +): string[] { + const matches = config.spec.sandboxes.every((sandbox) => + sandbox.agents.every((agent) => + agent.inference.routes.every( + (route) => + route.providerRef !== provider.name || + isDeepStrictEqual( + [sandbox.runtime.provider, route.overrides.model, route.overrides.contextWindow], + ["docker", provider.serving.model.servedName, EXPORTED_VLLM_CONTEXT_WINDOW], + ), + ), + ), + ); + return matches + ? [] + : [ + `/spec/inferenceProviders/${providerIndex}/serving does not match the sandbox runtime or route model`, + ]; +} + function semanticProblems(config: NemoClawConfig): string[] { const problems = [ ...duplicateProblems( @@ -109,6 +158,10 @@ function semanticProblems(config: NemoClawConfig): string[] { ]; const providers = new Set(config.spec.inferenceProviders.map(({ name }) => name)); for (const [providerIndex, provider] of config.spec.inferenceProviders.entries()) { + if ("serving" in provider) { + problems.push(...managedProviderProblems(config, provider, providerIndex)); + continue; + } const endpointViolation = unsafeEndpointUrlViolation(provider.endpoint); if (endpointViolation) problems.push( diff --git a/src/lib/domain/config/export-document.test.ts b/src/lib/domain/config/export-document.test.ts index 2e81ba05c07..721015f402d 100644 --- a/src/lib/domain/config/export-document.test.ts +++ b/src/lib/domain/config/export-document.test.ts @@ -99,6 +99,27 @@ describe("export config builder", () => { }); }); + it("emits a verified Brave integration without another inference provider (#10904)", () => { + const webSearch = { + provider: "brave", + agentRefs: ["primary"], + credential: { env: "BRAVE_API_KEY" }, + } as const; + const document = buildExportConfig( + { ...source, webSearch }, + { + documentName: alphaDocumentName, + documentUid: firstUid, + }, + ); + expect(document.spec.sandboxes[0]!.integrations).toEqual({ webSearch }); + expect(document.spec.inferenceProviders).toHaveLength(1); + expect( + buildExportConfig(source, { documentName: alphaDocumentName, documentUid: firstUid }).spec + .sandboxes[0], + ).not.toHaveProperty("integrations"); + }); + it("uses the supplied identity and keeps derived references deterministic (#10938)", () => { const first = buildExportConfig(source, { documentName: alphaDocumentName, diff --git a/src/lib/domain/config/export-document.ts b/src/lib/domain/config/export-document.ts index f4356130103..888519bd98f 100644 --- a/src/lib/domain/config/export-document.ts +++ b/src/lib/domain/config/export-document.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { EXPORTED_VLLM_CONTEXT_WINDOW } from "../../config/model"; import type { NemoClawConfig, NemoClawConfigDocumentName, @@ -21,6 +22,14 @@ function inferenceProvider( source: VerifiedExportSource, name: string, ): NemoClawInferenceProviderConfig { + if ("serving" in source.inference) { + return { + name, + provider: source.inference.provider, + api: source.inference.api, + serving: source.inference.serving, + }; + } const provider = { name, provider: source.inference.provider, @@ -42,7 +51,8 @@ export function buildExportConfig( source: VerifiedExportSource, identity: ExportConfigBuildIdentity, ): NemoClawConfig { - const providerName = providerLocalName(source.inference.provider); + const providerName = + "serving" in source.inference ? "managed-vllm" : providerLocalName(source.inference.provider); const candidate = { apiVersion: "nemoclaw.nvidia.com/v1", kind: "NemoClawConfig", @@ -65,6 +75,9 @@ export function buildExportConfig( policy: { explicit: source.policy }, ...(source.proxy === undefined ? {} : { proxy: source.proxy }), }, + ...(source.webSearch === undefined + ? {} + : { integrations: { webSearch: source.webSearch } }), agents: [ { name: "primary", @@ -74,7 +87,12 @@ export function buildExportConfig( { name: "primary", providerRef: providerName, - overrides: { model: source.inference.model }, + overrides: { + model: source.inference.model, + ...("serving" in source.inference + ? { contextWindow: EXPORTED_VLLM_CONTEXT_WINDOW } + : {}), + }, }, ], }, diff --git a/src/lib/domain/config/export-evidence.ts b/src/lib/domain/config/export-evidence.ts index 858bec9068a..b46a63dc95f 100644 --- a/src/lib/domain/config/export-evidence.ts +++ b/src/lib/domain/config/export-evidence.ts @@ -4,11 +4,13 @@ import type * as TypeBoxModule from "typebox" with { "resolution-mode": "import" }; import { BoundedTextSchema, + NemoClawManagedVllmServingSchema, CredentialEnvironmentReferenceNameSchema, ImmutableImageReferenceSchema, InferenceEndpointSchema, LocalResourceNameSchema, NemoClawInferenceApiSchema, + NemoClawBraveSearchConfigSchema, NemoClawAgentTypeSchema, NemoClawManagedProxyConfigSchema, RuntimeProviderSchema, @@ -70,6 +72,7 @@ export const EXPORT_REGISTRY_EVIDENCE_KEYS = [ "provider", "sandboxGpuDevice", "sandboxGpuEnabled", + "servingProfileProvenance", "toolDisclosure", "webSearchEnabled", "webSearchProvider", @@ -99,6 +102,13 @@ export interface ObservedExportEndpointEvidence { readonly name: string; readonly id: string; readonly resourceVersion: string; + readonly profileWorkspace?: string; + readonly managedProfile?: { + readonly id: "brave" | "openai"; + readonly source: "builtin" | "user"; + readonly scope: "" | "platform" | "workspace"; + readonly resourceVersion: string; + }; }; readonly endpoint: string; readonly source: @@ -109,6 +119,32 @@ export interface ObservedExportEndpointEvidence { | { readonly kind: "builtin-profile"; readonly profileId: "nvidia" }; } +export interface ObservedExportWebSearchProvider { + readonly gatewayName: string; + readonly workspace: string; + readonly name: string; + readonly id: string; + readonly resourceVersion: string; + readonly type: string; + readonly credentialKeys: readonly string[]; + readonly configKeys: readonly string[]; + readonly profileWorkspace?: string; + readonly profile?: { + readonly id: string; + readonly source: string; + readonly scope: string; + readonly resourceVersion: string; + }; +} + +export interface ObservedManagedVllmRuntime { + readonly serving: import("../../config/model").NemoClawManagedVllmServing; + readonly containerId: string; + readonly imageId: string; + readonly networkId: string; + readonly startedAt: string; +} + export interface ObservedExportInference { readonly topology: "hosted" | "managed" | "local" | "unknown"; readonly provider: string; @@ -118,6 +154,7 @@ export interface ObservedExportInference { readonly endpoint: string; readonly endpointEvidence: ObservedExportEndpointEvidence | null; readonly credentialEnv: string | null; + readonly managedServing?: ObservedManagedVllmRuntime; } export interface ObservedExportPolicy { @@ -143,6 +180,8 @@ export type ExportSnapshotReadStage = | "sandbox-identity" | "inference-route" | "provider-metadata" + | "web-search-provider" + | "managed-serving" | "effective-policy"; /** One complete, untrusted read from all export evidence owners. */ @@ -159,6 +198,7 @@ export type RawExportSnapshot = sandbox: ObservedExportSandboxIdentity; gateway: ObservedExportGateway; inference: ObservedExportInference; + webSearchProvider?: ObservedExportWebSearchProvider; policy: ObservedExportPolicy; configuration: SandboxConfiguration; }>; @@ -194,8 +234,11 @@ export interface ExportFinding { export type NonEmptyExportFindings = readonly [ExportFinding, ...ExportFinding[]]; // Runtime refinements preserve semantic checks that are not part of JSON Schema. -const ExportInferenceSchema = Type.Object({ - provider: Type.Refine(BoundedTextSchema, isValidNemoClawBoundedText), +const HostedExportInferenceSchema = Type.Object({ + provider: Type.Refine( + BoundedTextSchema, + (value) => isValidNemoClawBoundedText(value) && value !== "vllm-local", + ), model: Type.Refine(BoundedTextSchema, isValidNemoClawBoundedText), api: NemoClawInferenceApiSchema, endpoint: Type.Refine(InferenceEndpointSchema, isValidNemoClawInferenceEndpoint), @@ -204,6 +247,19 @@ const ExportInferenceSchema = Type.Object({ ), }); +const ExportInferenceSchema = Type.Union([ + HostedExportInferenceSchema, + Type.Object( + { + provider: Type.Literal("vllm-local"), + model: Type.Refine(BoundedTextSchema, isValidNemoClawBoundedText), + api: Type.Literal("openai-completions"), + serving: NemoClawManagedVllmServingSchema, + }, + { additionalProperties: false }, + ), +]); + /** Representable values only; provenance and policy qualification remain separate. */ export const ExportSourceValuesSchema = Type.Object({ sandboxName: Type.Refine(SandboxNameSchema, isValidNemoClawSandboxName), @@ -215,6 +271,7 @@ export const ExportSourceValuesSchema = Type.Object({ gateway: Type.Object({ name: LocalResourceNameSchema, port: TcpPortSchema }), proxy: Type.Optional(NemoClawManagedProxyConfigSchema), inference: ExportInferenceSchema, + webSearch: Type.Optional(NemoClawBraveSearchConfigSchema), }); type ExportSourceValues = DeepReadonly>; diff --git a/src/lib/domain/config/verify-export-source.test.ts b/src/lib/domain/config/verify-export-source.test.ts index 9821437430f..a6c555e12dd 100644 --- a/src/lib/domain/config/verify-export-source.test.ts +++ b/src/lib/domain/config/verify-export-source.test.ts @@ -201,6 +201,33 @@ function snapshot(overrides: Partial = {}): ObservedExpo }; } +function braveSnapshot(): ObservedExportSnapshot { + const value = snapshot(); + return { + ...value, + registry: entry({ + webSearchEnabled: true, + webSearchProvider: "brave", + workload: managedWorkload( + profileInput({ webSearch: { fetchEnabled: true, provider: "brave" } }), + ), + }), + sandbox: { ...value.sandbox, providerNames: ["alpha-brave-search"] }, + webSearchProvider: { + gatewayName: "nemoclaw", + workspace: "default", + name: "alpha-brave-search", + id: "brave-provider-id", + resourceVersion: "4", + type: "brave", + profileWorkspace: "default", + profile: { id: "brave", source: "user", scope: "workspace", resourceVersion: "4" }, + credentialKeys: ["BRAVE_API_KEY"], + configKeys: [], + }, + }; +} + function hermesSnapshot(registryOverrides: Partial = {}): ObservedExportSnapshot { const workload = managedWorkload(hermesProfileInput(), hermesImageRef); return snapshot({ @@ -270,6 +297,97 @@ function proxySnapshot( } describe("config export source verification (#10938)", () => { + it.each([false, true])( + "verifies Brave with optional inference attachment %s (#10904)", + (attached) => { + const value = braveSnapshot(); + const providers = attached + ? [value.inference.provider, "alpha-brave-search"] + : ["alpha-brave-search"]; + const result = verify({ ...value, sandbox: { ...value.sandbox, providerNames: providers } }); + expect(verifiedSource(result).webSearch).toEqual({ + provider: "brave", + agentRefs: ["primary"], + credential: { env: "BRAVE_API_KEY" }, + }); + }, + ); + + it.each([ + [], + ["foreign-brave-search"], + ["alpha-brave-search", "alpha-brave-search"], + ["alpha-brave-search", "extra"], + ["alpha-brave-search", "openai-api", "openai-api"], + ])("rejects missing or unexpected Brave attachments %j (#10904)", (...providerNames) => { + const value = braveSnapshot(); + expect( + findings(verify({ ...value, sandbox: { ...value.sandbox, providerNames } })), + ).toContainEqual( + expect.objectContaining({ field: "source.sandbox.providers", category: "unsupported" }), + ); + }); + + it.each([ + { gatewayName: "foreign" }, + { workspace: "foreign" }, + { name: "foreign-brave-search" }, + { id: "" }, + { resourceVersion: "" }, + { resourceVersion: "0" }, + { type: "generic" }, + { credentialKeys: ["TAVILY_API_KEY"] }, + { credentialKeys: ["BRAVE_API_KEY", "OTHER_KEY"] }, + { configKeys: ["BASE_URL"] }, + { profileWorkspace: "foreign" }, + { profileWorkspace: undefined }, + { profile: undefined }, + { profile: { id: "other", source: "builtin", scope: "", resourceVersion: "0" } }, + { profile: { id: "brave", source: "user", scope: "platform", resourceVersion: "1" } }, + { profile: { id: "brave", source: "builtin", scope: "workspace", resourceVersion: "0" } }, + { profile: { id: "brave", source: "builtin", scope: "", resourceVersion: "1" } }, + ])("rejects mismatched Brave metadata %j (#10904)", (change) => { + const value = braveSnapshot(); + expect( + findings(verify({ ...value, webSearchProvider: { ...value.webSearchProvider!, ...change } })), + ).toContainEqual(expect.objectContaining({ field: "source.webSearch", category: "drifted" })); + }); + + it("requires Brave metadata and matching startup intent (#10904)", () => { + const value = braveSnapshot(); + expect(findings(verify({ ...value, webSearchProvider: undefined }))).toContainEqual( + expect.objectContaining({ field: "source.webSearch", category: "missing-provenance" }), + ); + expect( + findings(verify({ ...value, registry: { ...value.registry, workload: managedWorkload() } })), + ).toContainEqual( + expect.objectContaining({ field: "source.workload.startupProfile", category: "unsupported" }), + ); + expect( + findings(verify({ ...value, registry: { ...value.registry, webSearchProvider: "tavily" } })), + ).toContainEqual( + expect.objectContaining({ + field: "spec.sandboxes[].integrations.webSearch", + category: "unsupported", + }), + ); + }); + + it("keeps unrelated startup settings unsupported with Brave enabled (#10904)", () => { + const value = braveSnapshot(); + const workload = managedWorkload( + profileInput({ + webSearch: { fetchEnabled: true, provider: "brave" }, + environment: { NEMOCLAW_AGENT_TIMEOUT: "900" }, + }), + ); + expect( + findings(verify({ ...value, registry: { ...value.registry, workload } })), + ).toContainEqual( + expect.objectContaining({ field: "source.workload.startupProfile", category: "unsupported" }), + ); + }); + it("qualifies and verifies two equal snapshots through the observer", async () => { const observed = snapshot(); const result = await observeStableExportSource("alpha", { @@ -442,6 +560,7 @@ describe("config export source verification (#10938)", () => { { runtime: { provider: "docker", imageRef: "registry/image:latest" } }, { gateway: { name: "nemoclaw", port: 0 } }, { inference: { provider: "e\u0301".repeat(257) } }, + { inference: { provider: "vllm-local" } }, { inference: { api: "openai-unknown" } }, { inference: { endpoint: "https://user:secret@api.example.com/v1" } }, { inference: { endpoint: "https://api.example.com/%0A%" } }, diff --git a/src/lib/domain/config/verify-export-source.ts b/src/lib/domain/config/verify-export-source.ts index 56e9804d1de..6d67b0b8cdb 100644 --- a/src/lib/domain/config/verify-export-source.ts +++ b/src/lib/domain/config/verify-export-source.ts @@ -16,6 +16,8 @@ import { readManagedWorkloadAuthority } from "../../onboard/workload/authority"; import { sortCanonicalMappings } from "../../config/canonical-mapping"; import { isCredentialEnvironmentReferenceName, + EXPORTED_VLLM_PROFILE_ID, + EXPORTED_VLLM_CONTEXT_WINDOW, isImmutableImageReference, isValidNemoClawBoundedText, isValidNemoClawInferenceEndpoint, @@ -27,6 +29,7 @@ import { } from "../../config/model"; import { fingerprintOpenShellSandboxId } from "../sandbox/openshell-identity"; import { ExportSourceValuesSchema } from "./export-evidence"; +import { validateManagedServing } from "./verify-managed-serving"; import type { CanonicalExportPolicy, ExportFinding, @@ -85,7 +88,7 @@ const EXPORT_AGENT_PROFILE_PROJECTIONS: Record< type VerifiedExportSourceData = Pick< VerifiedExportSource, - "agent" | "gateway" | "inference" | "policy" | "proxy" | "runtime" | "sandboxName" + "agent" | "gateway" | "inference" | "policy" | "proxy" | "runtime" | "sandboxName" | "webSearch" >; function verifiedExportSource(data: VerifiedExportSourceData): VerifiedExportSource { @@ -118,6 +121,10 @@ function hasEntries(value: unknown): boolean { : value !== undefined && value !== null && value !== false; } +function hasBraveSearch(entry: ObservedExportRegistry): boolean { + return entry.webSearchEnabled === true && entry.webSearchProvider === "brave"; +} + function classifyHermesExcludedCapabilities(entry: ObservedExportRegistry): ExportFinding[] { const excluded: Array<[string, unknown, string]> = [ ["spec.sandboxes[].agents[0].tools", entry.hermesToolGateways, "enabled Hermes tool gateways"], @@ -170,7 +177,7 @@ function classifyExcludedCapabilities(entry: ObservedExportRegistry): ExportFind ["spec.sandboxes[].observability", entry.observabilityEnabled, "observability"], [ "spec.sandboxes[].integrations.webSearch", - entry.webSearchEnabled || entry.webSearchProvider, + !hasBraveSearch(entry) && (entry.webSearchEnabled || entry.webSearchProvider), "web search", ], ["spec.sandboxes[].integrations.messaging", entry.messaging, "messaging"], @@ -340,7 +347,7 @@ export function classifyExportRegistry(entry: ObservedExportRegistry): ExportFin finding( "spec.inferenceProviders", "unsupported", - "V1 export supports hosted external inference only.", + "This local inference topology is not represented by v1 export.", ), ); return findings; @@ -380,13 +387,16 @@ function expectedManagedStartupProfile(entry: ObservedExportRegistry): ManagedSt compatibility: projection.compatibility, }, dashboard: projection.dashboard, - webSearch: null, + webSearch: hasBraveSearch(entry) ? { fetchEnabled: true, provider: "brave" } : null, toolDisclosure: "progressive", hermesToolGateways: [], messagingPlan: null, dcodeAutoApprovalMode: null, observabilityEnabled: null, - environment: {}, + environment: + entry.servingProfileProvenance?.preset.id === EXPORTED_VLLM_PROFILE_ID + ? { NEMOCLAW_CONTEXT_WINDOW: String(EXPORTED_VLLM_CONTEXT_WINDOW) } + : {}, corporateCa: null, }).profile; } @@ -530,17 +540,93 @@ function validateSandboxConfiguration(snapshot: QualifiedExportSnapshot): Export "Registry and live sandbox images differ.", ), ); - if (sandbox.providerNames.some((name) => name !== inference.provider)) + const additionalProviders = sandbox.providerNames.filter((name) => name !== inference.provider); + const expectedAdditionalProviders = hasBraveSearch(entry) ? [`${entry.name}-brave-search`] : []; + if ( + !isDeepStrictEqual(additionalProviders, expectedAdditionalProviders) || + new Set(sandbox.providerNames).size !== sandbox.providerNames.length + ) findings.push( finding( "source.sandbox.providers", "unsupported", - "V1 export does not support additional provider attachments.", + "The sandbox provider attachments do not match its supported configuration.", ), ); return findings; } +function validBraveProfile( + provider: NonNullable, +): boolean { + const { profile, profileWorkspace } = provider; + if (!profile || profile.id !== "brave" || !isValidNemoClawBoundedText(profile.resourceVersion)) + return false; + if (profile.source === "builtin") { + return isDeepStrictEqual( + [profileWorkspace, profile.scope, profile.resourceVersion], + ["", "", "0"], + ); + } + return ( + profile.source === "user" && + /^[1-9][0-9]*$/u.test(profile.resourceVersion) && + (isDeepStrictEqual([profileWorkspace, profile.scope], ["", "platform"]) || + isDeepStrictEqual([profileWorkspace, profile.scope], [provider.workspace, "workspace"])) + ); +} + +function validateWebSearchProvider(snapshot: QualifiedExportSnapshot): ExportFinding[] { + const { registry, webSearchProvider: provider, sandbox, gateway } = snapshot; + if (!hasBraveSearch(registry)) { + return provider === undefined + ? [] + : [finding("source.webSearch", "ambiguous", "Unexpected web-search provider evidence.")]; + } + if (!provider) { + return [ + finding( + "source.webSearch", + "missing-provenance", + "Live Brave provider evidence is required.", + ), + ]; + } + if ( + !validBraveProfile(provider) || + !isValidNemoClawBoundedText(provider.id) || + !isValidNemoClawBoundedText(provider.resourceVersion) || + !/^[1-9][0-9]*$/u.test(provider.resourceVersion) || + !isDeepStrictEqual( + [ + provider.gatewayName, + provider.workspace, + provider.name, + provider.type, + provider.credentialKeys, + provider.configKeys, + ], + [ + gateway.name, + sandbox.workspace, + `${registry.name}-brave-search`, + "brave", + ["BRAVE_API_KEY"], + [], + ], + ) + ) { + return [ + finding( + "source.webSearch", + "drifted", + "The live Brave provider does not match its managed binding.", + ), + ]; + } + return []; +} + function validateGateway(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { registry: entry, gateway } = snapshot; const findings: ExportFinding[] = []; @@ -569,15 +655,26 @@ function validateGateway(snapshot: QualifiedExportSnapshot): ExportFinding[] { function validateInferenceSelection(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { registry: entry, inference } = snapshot; const findings: ExportFinding[] = []; - if (inference.topology !== "hosted") + if (inference.topology !== "hosted" && inference.topology !== "managed") findings.push( finding( "spec.inferenceProviders", "unsupported", - "V1 export supports hosted external inference only.", + "This local inference topology is not represented by v1 export.", ), ); + if ( + inference.topology !== "managed" && + (entry.servingProfileProvenance || inference.managedServing) + ) + findings.push( + finding( + "spec.inferenceProviders[].serving", + "unsupported", + "Recorded managed serving requires complete live managed runtime evidence.", + ), + ); const selected = normalizeInferenceSelection(entry); if ( !isDeepStrictEqual( @@ -609,6 +706,7 @@ function validateInferenceSelection(snapshot: QualifiedExportSnapshot): ExportFi function validateInferenceRepresentation(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { inference } = snapshot; + if (inference.topology === "managed") return validateManagedServing(snapshot); const findings: ExportFinding[] = []; if ( [inference.provider, inference.model, inference.api, inference.endpoint].some((value) => !value) @@ -658,7 +756,7 @@ function validateEndpointEvidence(snapshot: QualifiedExportSnapshot): ExportFind } const findings: ExportFinding[] = []; - if (!isValidNemoClawInferenceEndpoint(evidence.endpoint)) + if (inference.topology !== "managed" && !isValidNemoClawInferenceEndpoint(evidence.endpoint)) findings.push( finding( "source.inference.endpoint", @@ -695,6 +793,7 @@ function validateEndpointEvidence(snapshot: QualifiedExportSnapshot): ExportFind function validateCredentialReference(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { inference } = snapshot; + if (inference.topology === "managed") return []; const findings: ExportFinding[] = []; if ( inference.credentialEnv !== null && @@ -753,6 +852,7 @@ function validateAgreement( ...classifyExportRegistry(snapshot.registry), ...validateSandboxIdentity(requestedSandboxName, snapshot), ...validateSandboxConfiguration(snapshot), + ...validateWebSearchProvider(snapshot), ...validateGateway(snapshot), ...validateInferenceSelection(snapshot), ...validateInferenceRepresentation(snapshot), @@ -794,18 +894,35 @@ function completeVerifiedSource( const values = { sandboxName: requestedSandboxName, agent: entry.agent, + ...(hasBraveSearch(entry) + ? { + webSearch: { + provider: "brave", + agentRefs: ["primary"], + credential: { env: "BRAVE_API_KEY" }, + }, + } + : {}), runtime: { provider: entry.openshellDriver, imageRef: authority?.receipt.reference }, gateway: { name: snapshot.gateway.name, port: snapshot.gateway.port }, ...(proxy && !hasEqualJsonStructure(proxy, expectedManagedStartupProfile(entry).proxy) ? { proxy: { host: proxy.managedHost, port: proxy.managedPort } } : {}), - inference: { - provider: selected.provider, - model: selected.model, - api: selected.preferredInferenceApi, - endpoint: selected.endpointUrl, - ...(selected.credentialEnv === null ? {} : { credentialEnv: selected.credentialEnv }), - }, + inference: + snapshot.inference.topology === "managed" + ? { + provider: selected.provider, + model: selected.model, + api: selected.preferredInferenceApi, + serving: snapshot.inference.managedServing?.serving, + } + : { + provider: selected.provider, + model: selected.model, + api: selected.preferredInferenceApi, + endpoint: selected.endpointUrl, + ...(selected.credentialEnv === null ? {} : { credentialEnv: selected.credentialEnv }), + }, }; if (!Check(ExportSourceValuesSchema, values)) { return { diff --git a/src/lib/domain/config/verify-managed-serving.ts b/src/lib/domain/config/verify-managed-serving.ts new file mode 100644 index 00000000000..9752156b9f0 --- /dev/null +++ b/src/lib/domain/config/verify-managed-serving.ts @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; +import type * as TypeBoxValueModule from "typebox/value" with { "resolution-mode": "import" }; +import { + EXPORTED_VLLM_PROFILE_ID, + EXPORTED_VLLM_RECIPE_ID, + NemoClawManagedVllmServingSchema, + type NemoClawManagedVllmServing, +} from "../../config/model"; +import type { ServingProfileProvenance } from "../../inference/serving/types"; +import type { + ExportFinding, + QualifiedExportSnapshot, + ObservedManagedVllmRuntime, + ObservedExportEndpointEvidence, +} from "./export-evidence"; + +const { Check } = require("typebox/value") as typeof TypeBoxValueModule; + +function validRuntimeIdentity(observed: ObservedManagedVllmRuntime): boolean { + return ( + Check(NemoClawManagedVllmServingSchema, observed.serving) && + /^[a-f0-9]{64}$/.test(observed.containerId) && + /^sha256:[a-f0-9]{64}$/.test(observed.imageId) && + /^[a-f0-9]{64}$/.test(observed.networkId) && + observed.startedAt.length > 0 && + observed.startedAt.length <= 64 + ); +} + +function matchesProvenance( + serving: NemoClawManagedVllmServing, + recorded: ServingProfileProvenance, +): boolean { + return isDeepStrictEqual( + [ + recorded.schemaVersion, + recorded.catalogDigest, + recorded.preset.id, + recorded.preset.digest, + recorded.recipe.id, + recorded.recipe.digest, + recorded.recipe.backend, + recorded.model, + recorded.runtimeImage, + ], + [ + 1, + serving.catalogDigest, + EXPORTED_VLLM_PROFILE_ID, + serving.profile.digest, + EXPORTED_VLLM_RECIPE_ID, + serving.recipe.digest, + "vllm", + { id: serving.model.id, revision: serving.model.revision }, + serving.runtime.image.ref, + ], + ); +} + +function validProfileVersion(version: string): boolean { + return /^(0|[1-9][0-9]{0,19})$/.test(version) && BigInt(version) <= 18446744073709551615n; +} + +function hasManagedOpenAiProfile(evidence: ObservedExportEndpointEvidence | null): boolean { + if (!evidence) return false; + const { provider } = evidence; + const profile = provider.managedProfile; + if (!profile) return false; + const version = profile.resourceVersion; + if (profile.id !== "openai" || !validProfileVersion(version)) return false; + if (profile.source === "builtin") + return isDeepStrictEqual([provider.profileWorkspace, profile.scope, version], ["", "", "0"]); + return ( + profile.source === "user" && + version !== "0" && + [ + ["", "platform"], + [provider.workspace, "workspace"], + ].some((binding) => isDeepStrictEqual([provider.profileWorkspace, profile.scope], binding)) + ); +} + +export function validateManagedServing(snapshot: QualifiedExportSnapshot): ExportFinding[] { + const { registry: entry, inference } = snapshot; + const observed = inference.managedServing; + const recorded = entry.servingProfileProvenance; + const invalid = () => [ + { + field: "spec.inferenceProviders[].serving", + category: "drifted" as const, + diagnostic: "The fixed managed serving identity, provenance, or route could not be verified.", + }, + ]; + if (!observed || !recorded) return invalid(); + const serving = observed.serving; + if ( + !validRuntimeIdentity(observed) || + !isDeepStrictEqual( + [ + entry.agent, + entry.openshellDriver, + entry.workload?.kind, + entry.workload?.kind === "managed-image" ? entry.workload.platform : null, + snapshot.sandbox.providerNames.filter((name) => name === inference.provider), + inference.provider, + inference.api, + inference.credentialEnv, + inference.model, + inference.endpoint, + ], + [ + "openclaw", + "docker", + "managed-image", + "linux/amd64", + ["vllm-local"], + "vllm-local", + "openai-completions", + null, + serving.model.servedName, + `http://host.openshell.internal:${String(serving.hostPort)}/v1`, + ], + ) || + !hasManagedOpenAiProfile(inference.endpointEvidence) || + !matchesProvenance(observed.serving, recorded) + ) + return invalid(); + return []; +} diff --git a/src/lib/inference/config.ts b/src/lib/inference/config.ts index 4c2b5e5da8a..32d86014f11 100644 --- a/src/lib/inference/config.ts +++ b/src/lib/inference/config.ts @@ -17,6 +17,7 @@ import type { ManagedLlamaCppOwnership } from "./llama-cpp/managed-state"; import { DEFAULT_OLLAMA_MODEL_TAG as DEFAULT_OLLAMA_MODEL } from "./ollama-model-registry"; import { OLLAMA_LOCAL_CREDENTIAL_ENV } from "./ollama/contract"; import { OPENROUTER_CREDENTIAL_ENV, OPENROUTER_PROVIDER_NAME } from "./openrouter"; +import { VLLM_LOCAL_CREDENTIAL_ENV } from "./serving/vllm-credential-contract"; export { isSafeModelId }; export { OLLAMA_LOCAL_CREDENTIAL_ENV }; @@ -75,7 +76,7 @@ export const DEFAULT_ROUTE_CREDENTIAL_ENV = "OPENAI_API_KEY"; // Dedicated credential env names for local inference. Decoupled from // OPENAI_API_KEY so the sandbox-side OpenClaw and the host-side gateway // never read the user's host OpenAI key for local providers. See GH #2519. -export const VLLM_LOCAL_CREDENTIAL_ENV = "NEMOCLAW_VLLM_LOCAL_TOKEN"; +export { VLLM_LOCAL_CREDENTIAL_ENV }; export const LLAMA_CPP_LOCAL_CREDENTIAL_ENV = LLAMA_CPP_CREDENTIAL_ENV; export const MANAGED_PROVIDER_ID = "inference"; export { DEFAULT_OLLAMA_MODEL }; diff --git a/src/lib/inference/serving/host-local-vllm-selection.ts b/src/lib/inference/serving/host-local-vllm-selection.ts index b76ec8798ad..edb3a0897c6 100644 --- a/src/lib/inference/serving/host-local-vllm-selection.ts +++ b/src/lib/inference/serving/host-local-vllm-selection.ts @@ -6,7 +6,7 @@ import os from "node:os"; import { getBuildIdentity } from "../../core/version.js"; import { createHostReadinessReport } from "../../readiness/host.js"; import type { VllmProfile } from "../vllm.js"; -import type { VllmModelDef } from "../vllm-models.js"; +import type { VllmModelDef, VllmPlatform } from "../vllm-models.js"; import { VLLM_EXTRA_ARGS_ENV } from "../vllm-models.js"; import { HOST_LOCAL_VLLM_LIFECYCLE_REF, @@ -24,6 +24,7 @@ import type { HostLocalInferenceServingRecipe, ManagedInferenceReadinessSource, ResolvedHostLocalInferenceSelection, + VllmDirectInstallPolicy, } from "./types.js"; export interface MaterializedHostLocalVllmSelection { @@ -38,13 +39,8 @@ export type HostLocalVllmSelectionResult = | { readonly kind: "rejected"; readonly reason: string } | ({ readonly kind: "selected" } & MaterializedHostLocalVllmSelection); -function positiveIntegerArgument( - selection: ResolvedHostLocalInferenceSelection, - name: string, -): number { - const matches = selection.recipe.spec.serve?.arguments?.filter( - (argument) => argument.name === name, - ); +function positiveIntegerArgument(recipe: HostLocalInferenceServingRecipe, name: string): number { + const matches = recipe.spec.serve?.arguments?.filter((argument) => argument.name === name); const value = matches?.length === 1 ? matches[0]!.value : undefined; const parsed = typeof value === "number" @@ -58,6 +54,51 @@ function positiveIntegerArgument( return parsed; } +/** Materialize the same fixed model command for installation and read-only runtime verification. */ +export function materializeHostLocalVllmModel( + recipe: HostLocalInferenceServingRecipe, + directInstall: VllmDirectInstallPolicy, + platform: VllmPlatform, +): VllmModelDef { + const runtime = recipe.spec.runtime; + const serveEnvironment = { + ...runtime.environment, + HF_HOME: runtime.modelCache.target, + HF_HUB_OFFLINE: "1", + TRANSFORMERS_OFFLINE: "1", + }; + const gpuMemoryUtilization = hostLocalVllmGpuMemoryUtilization(recipe); + return { + id: recipe.spec.model.id, + label: recipe.spec.model.displayName, + envValue: recipe.spec.model.environmentValue, + downloadSizeBytes: recipe.spec.model.downloadSizeBytes, + maxModelLen: positiveIntegerArgument(recipe, "--max-model-len"), + revision: recipe.spec.model.revision, + servedModelId: recipe.spec.model.servedName, + modelArgs: hostLocalVllmModelArguments(recipe), + gated: recipe.spec.model.gated, + platforms: [platform], + minComputeCapability: runtime.minimumComputeCapability, + ...(Object.keys(serveEnvironment).length > 0 ? { serveEnv: serveEnvironment } : {}), + runtime: { + image: runtime.image, + imageDownloadSizeBytes: runtime.imageDownloadSizeBytes, + modelDownloadSizeBytes: recipe.spec.model.downloadSizeBytes, + loadTimeoutSec: recipe.spec.readiness.timeoutSeconds, + pullTimeoutSec: runtime.pullTimeoutSeconds, + minComputeCapability: runtime.minimumComputeCapability, + minGpuMemoryBytes: runtime.minimumGpuMemoryBytes, + gpuMemoryUtilization, + dockerRunArgs: hostLocalVllmDockerRunArguments(recipe), + dockerRunArgsMode: "replace", + }, + installFastSafetensors: recipe.spec.model.installFastSafetensors, + ...(directInstall.authentication === "bearer" ? { managedBearerAuth: true as const } : {}), + ...(directInstall.fixedArguments ? { fixedServeCommand: true as const } : {}), + }; +} + export function materializeHostLocalVllmSelection( selection: ResolvedHostLocalInferenceSelection, baseProfile: VllmProfile, @@ -96,42 +137,8 @@ export function materializeHostLocalVllmSelection( ) { throw new Error("host-local vLLM recipe is missing required runtime or model fields"); } - const serveEnvironment = { - ...runtime.environment, - HF_HOME: runtime.modelCache.target, - HF_HUB_OFFLINE: "1", - TRANSFORMERS_OFFLINE: "1", - }; + const model = materializeHostLocalVllmModel(recipe, directInstall, baseProfile.platform); const gpuMemoryUtilization = hostLocalVllmGpuMemoryUtilization(recipe); - const model: VllmModelDef = { - id: recipe.spec.model.id, - label: recipe.spec.model.displayName, - envValue: recipe.spec.model.environmentValue, - downloadSizeBytes: recipe.spec.model.downloadSizeBytes, - maxModelLen: positiveIntegerArgument(selection, "--max-model-len"), - revision: recipe.spec.model.revision, - servedModelId, - modelArgs: hostLocalVllmModelArguments(recipe), - gated: recipe.spec.model.gated, - platforms: [baseProfile.platform], - minComputeCapability: runtime.minimumComputeCapability, - ...(Object.keys(serveEnvironment).length > 0 ? { serveEnv: serveEnvironment } : {}), - runtime: { - image: runtime.image, - imageDownloadSizeBytes: runtime.imageDownloadSizeBytes, - modelDownloadSizeBytes: recipe.spec.model.downloadSizeBytes, - loadTimeoutSec: recipe.spec.readiness.timeoutSeconds, - pullTimeoutSec: runtime.pullTimeoutSeconds, - minComputeCapability: runtime.minimumComputeCapability, - minGpuMemoryBytes: runtime.minimumGpuMemoryBytes, - gpuMemoryUtilization, - dockerRunArgs: hostLocalVllmDockerRunArguments(recipe), - dockerRunArgsMode: "replace", - }, - installFastSafetensors: recipe.spec.model.installFastSafetensors, - ...(directInstall.authentication === "bearer" ? { managedBearerAuth: true as const } : {}), - ...(directInstall.fixedArguments ? { fixedServeCommand: true as const } : {}), - }; return { presetId: preset.metadata.id, recipeId: recipe.metadata.id, diff --git a/src/lib/inference/serving/vllm-credential-contract.ts b/src/lib/inference/serving/vllm-credential-contract.ts new file mode 100644 index 00000000000..2640e586135 --- /dev/null +++ b/src/lib/inference/serving/vllm-credential-contract.ts @@ -0,0 +1,5 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Internal OpenShell credential slot; never a public user credential reference. */ +export const VLLM_LOCAL_CREDENTIAL_ENV = "NEMOCLAW_VLLM_LOCAL_TOKEN"; diff --git a/src/lib/inference/serving/vllm-export-runtime.test.ts b/src/lib/inference/serving/vllm-export-runtime.test.ts new file mode 100644 index 00000000000..722fca8f947 --- /dev/null +++ b/src/lib/inference/serving/vllm-export-runtime.test.ts @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { EXPORTED_VLLM_PROFILE_ID } from "../../config/model"; +import { loadServingCatalog } from "./catalog-loader"; +import { servingProfileProvenance } from "./profile-provenance"; +import { runtimeAuthFingerprint } from "./runtime-auth-fingerprint"; +import { + HOST_LOCAL_VLLM_AUTH_LABEL, + HOST_LOCAL_VLLM_CATALOG_LABEL, + HOST_LOCAL_VLLM_MANAGED_LABEL, + HOST_LOCAL_VLLM_PRESET_DIGEST_LABEL, + HOST_LOCAL_VLLM_PRESET_LABEL, + HOST_LOCAL_VLLM_RECIPE_DIGEST_LABEL, + HOST_LOCAL_VLLM_RECIPE_LABEL, + persistHostLocalVllmRuntimeReceipt, +} from "./vllm-host-local-lifecycle"; +import { observeManagedVllmForExport, type VllmExportRuntimeOptions } from "./vllm-export-runtime"; + +const temporaryDirectories: string[] = []; +const key = "e".repeat(64); +const containerId = "a".repeat(64); +const imageId = `sha256:${"b".repeat(64)}`; +const profile = () => servingProfileProvenance(loadServingCatalog(), EXPORTED_VLLM_PROFILE_ID); + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) + fs.rmSync(directory, { recursive: true, force: true }); +}); + +function fixture() { + const provenance = profile(); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-vllm-export-")); + temporaryDirectories.push(directory); + const fingerprint = runtimeAuthFingerprint(key); + persistHostLocalVllmRuntimeReceipt( + { + containerId, + authFingerprint: fingerprint, + serving: { + catalogDigest: provenance.catalogDigest, + presetId: provenance.preset.id, + presetDigest: provenance.preset.digest, + recipeId: provenance.recipe.id, + recipeDigest: provenance.recipe.digest, + }, + }, + directory, + ); + const image = { Id: imageId, Os: "linux", Architecture: "amd64", Environment: ["PATH=/usr/bin"] }; + const network = { + Id: "c".repeat(64), + Name: "openshell-docker", + Driver: "bridge", + Config: [{ Gateway: "172.18.0.1", Subnet: "172.18.0.0/16" }], + }; + const container = { + Id: containerId, + Name: "/nemoclaw-vllm", + Image: imageId, + StartedAt: "2026-09-10T10:00:00Z", + Matches: true, + State: { Running: true }, + Config: { + Env: [`VLLM_API_KEY=${key}`], + Labels: { + [HOST_LOCAL_VLLM_AUTH_LABEL]: fingerprint, + [HOST_LOCAL_VLLM_MANAGED_LABEL]: "true", + [HOST_LOCAL_VLLM_CATALOG_LABEL]: provenance.catalogDigest, + [HOST_LOCAL_VLLM_PRESET_LABEL]: provenance.preset.id, + [HOST_LOCAL_VLLM_PRESET_DIGEST_LABEL]: provenance.preset.digest, + [HOST_LOCAL_VLLM_RECIPE_LABEL]: provenance.recipe.id, + [HOST_LOCAL_VLLM_RECIPE_DIGEST_LABEL]: provenance.recipe.digest, + }, + }, + NetworkSettings: { + Ports: { + "8000/tcp": [ + { HostIp: "127.0.0.1", HostPort: "18000" }, + { HostIp: "172.18.0.1", HostPort: "18000" }, + ], + }, + }, + }; + const capture = vi.fn>((args) => { + return JSON.stringify( + { image, network, container }[args[0] as "image" | "network" | "container"], + ); + }); + const loadApiKey = vi.fn(() => key); + const options = { + capture, + platform: "linux", + architecture: "x64", + authentication: { stateDir: directory, loadApiKey }, + }; + return { provenance, directory, image, network, container, capture, loadApiKey, options }; +} + +describe("fixed managed vLLM export observation", () => { + it("binds one running runtime to the current catalog and nondefault listener", () => { + const f = fixture(); + const result = observeManagedVllmForExport(f.provenance, f.options); + expect(result).toMatchObject({ + containerId, + imageId, + networkId: f.network.Id, + startedAt: f.container.StartedAt, + serving: { + profile: { id: EXPORTED_VLLM_PROFILE_ID }, + model: f.provenance.model, + hostPort: 18000, + }, + }); + expect(f.loadApiKey).toHaveBeenCalledOnce(); + const serialized = JSON.stringify(result); + expect(serialized).not.toContain(key); + expect(serialized).not.toContain(runtimeAuthFingerprint(key)); + expect(serialized).not.toContain("VLLM_API_KEY"); + expect(serialized).not.toContain(f.directory); + expect(f.capture.mock.calls.map(([args]) => args.slice(0, 2))).toEqual([ + ["image", "inspect"], + ["network", "inspect"], + ["container", "inspect"], + ]); + expect(f.capture.mock.calls.map(([args]) => args[2])).toEqual([ + "--format", + "--format", + "--format", + ]); + expect(JSON.stringify(f.capture.mock.calls.map(([args]) => args))).not.toContain(key); + const boundedCapture = expect.objectContaining({ + env: expect.objectContaining({ DOCKER_CONTEXT: "default" }), + timeout: 5000, + maxBuffer: 65536, + }); + expect(f.capture.mock.calls.map(([, options]) => options)).toEqual([ + boundedCapture, + boundedCapture, + boundedCapture, + ]); + }); + + it.each([ + [ + "foreign container", + (f: ReturnType) => { + f.container.Id = "d".repeat(64); + }, + ], + [ + "stopped container", + (f: ReturnType) => { + f.container.State.Running = false; + }, + ], + [ + "changed fixed configuration", + (f: ReturnType) => { + f.container.Matches = false; + }, + ], + [ + "substituted image", + (f: ReturnType) => { + f.container.Image = `sha256:${"d".repeat(64)}`; + }, + ], + [ + "foreign serving label", + (f: ReturnType) => { + f.container.Config.Labels[HOST_LOCAL_VLLM_RECIPE_DIGEST_LABEL] = `sha256:${"d".repeat(64)}`; + }, + ], + [ + "mismatched token", + (f: ReturnType) => { + f.container.Config.Env = [`VLLM_API_KEY=${"d".repeat(64)}`]; + }, + ], + [ + "additional credential payload", + (f: ReturnType) => { + f.container.Config.Env.push("SECRET=private-value"); + }, + ], + [ + "public listener", + (f: ReturnType) => { + f.container.NetworkSettings.Ports["8000/tcp"][0]!.HostIp = "0.0.0.0"; + }, + ], + [ + "different bridge listener", + (f: ReturnType) => { + f.network.Config[0]!.Gateway = "172.19.0.1"; + }, + ], + [ + "additional port", + (f: ReturnType) => { + Object.assign(f.container.NetworkSettings.Ports, { "9000/tcp": [] }); + }, + ], + [ + "unsafe bridge", + (f: ReturnType) => { + f.network.Config[0]!.Gateway = "169.254.169.254"; + }, + ], + [ + "wrong architecture", + (f: ReturnType) => { + f.image.Architecture = "arm64"; + }, + ], + [ + "duplicate image environment", + (f: ReturnType) => { + f.image.Environment.push(f.image.Environment[0]!); + }, + ], + [ + "missing receipt", + (f: ReturnType) => { + fs.unlinkSync(path.join(f.directory, "host-local-vllm-runtime.json")); + }, + ], + [ + "oversized receipt", + (f: ReturnType) => { + fs.writeFileSync(path.join(f.directory, "host-local-vllm-runtime.json"), " ".repeat(65537)); + }, + ], + ])("rejects %s without leaking private observations", (_label, change) => { + const f = fixture(); + change(f); + expect(() => observeManagedVllmForExport(f.provenance, f.options)).toThrow( + "The fixed managed vLLM runtime could not be verified for export.", + ); + }); + + it.each(["{malformed", "x".repeat(65537)])("redacts failed or malformed inspection", (value) => { + const f = fixture(); + f.capture.mockReturnValue(value); + expect(() => observeManagedVllmForExport(f.provenance, f.options)).toThrow( + "The fixed managed vLLM runtime could not be verified for export.", + ); + expect(f.loadApiKey).not.toHaveBeenCalled(); + }); + + it("redacts runtime inspection failures before private authentication access", () => { + const f = fixture(); + f.capture.mockImplementation(() => { + throw new Error("timeout credential-canary"); + }); + expect(() => observeManagedVllmForExport(f.provenance, f.options)).toThrow( + "The fixed managed vLLM runtime could not be verified for export.", + ); + expect(f.loadApiKey).not.toHaveBeenCalled(); + }); + + it("rejects stale catalog identity before Docker and credential access", () => { + const f = fixture(); + expect(() => + observeManagedVllmForExport( + { ...f.provenance, catalogDigest: `sha256:${"f".repeat(64)}` }, + f.options, + ), + ).toThrow(); + expect(f.capture).not.toHaveBeenCalled(); + expect(f.loadApiKey).not.toHaveBeenCalled(); + }); + + it("rejects an unqualified host before runtime inspection", () => { + const f = fixture(); + expect(() => + observeManagedVllmForExport(f.provenance, { ...f.options, architecture: "arm64" }), + ).toThrow(); + expect(f.capture).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/inference/serving/vllm-export-runtime.ts b/src/lib/inference/serving/vllm-export-runtime.ts new file mode 100644 index 00000000000..217e536c8f5 --- /dev/null +++ b/src/lib/inference/serving/vllm-export-runtime.ts @@ -0,0 +1,315 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import os from "node:os"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import type * as TypeBoxModule from "typebox" with { "resolution-mode": "import" }; +import type * as TypeBoxValueModule from "typebox/value" with { "resolution-mode": "import" }; +import { dockerCapture } from "../../adapters/docker/local-model-runtime"; +import { + EXPORTED_VLLM_PROFILE_ID, + EXPORTED_VLLM_CONTEXT_WINDOW, + EXPORTED_VLLM_RECIPE_ID, + isImmutableImageReference, + type NemoClawManagedVllmServing, +} from "../../config/model"; +import type { ObservedManagedVllmRuntime } from "../../domain/config/export-evidence"; +import { buildVllmServeCommand } from "../vllm-models"; +import { buildLocalManagedVllmDockerEnv } from "../vllm-docker-env"; +import { isHostLocalInferenceServingRecipe } from "./adapter-registry"; +import { loadManagedInferenceCatalog, loadServingCatalog } from "./catalog-loader"; +import { materializeHostLocalVllmModel } from "./host-local-vllm-selection"; +import { assertServingProfileProvenanceCurrent } from "./profile-provenance"; +import type { ServingProfileProvenance } from "./types"; +import { + HOST_LOCAL_VLLM_AUTH_LABEL, + HOST_LOCAL_VLLM_CATALOG_LABEL, + HOST_LOCAL_VLLM_CONTAINER_NAME, + HOST_LOCAL_VLLM_MANAGED_LABEL, + HOST_LOCAL_VLLM_PRESET_DIGEST_LABEL, + HOST_LOCAL_VLLM_PRESET_LABEL, + HOST_LOCAL_VLLM_RECIPE_DIGEST_LABEL, + HOST_LOCAL_VLLM_RECIPE_LABEL, + recoverHostLocalManagedVllmEndpoint, + type RecoverHostLocalManagedVllmOptions, +} from "./vllm-host-local-lifecycle"; +import { validateManagedVllmBridgeHost } from "./vllm-host-local-network"; + +const { Type } = require("typebox") as typeof TypeBoxModule; +const { Check } = require("typebox/value") as typeof TypeBoxValueModule; +const MAX_INSPECTION_BYTES = 64 * 1024; +const INSPECTION_TIMEOUT_MS = 5_000; +const Id = Type.String({ pattern: "^[a-f0-9]{64}$" }); +const ImageId = Type.String({ pattern: "^sha256:[a-f0-9]{64}$" }); +const Text = Type.String({ maxLength: 8192 }); +const ImageSchema = Type.Object( + { + Id: ImageId, + Os: Type.Literal("linux"), + Architecture: Type.Literal("amd64"), + Environment: Type.Array(Text, { maxItems: 128 }), + }, + { additionalProperties: false }, +); +const NetworkSchema = Type.Object( + { + Id, + Name: Type.Literal("openshell-docker"), + Driver: Type.Literal("bridge"), + Config: Type.Array(Type.Object({ Gateway: Text, Subnet: Text }), { minItems: 1, maxItems: 1 }), + }, + { additionalProperties: false }, +); +const ContainerSchema = Type.Object( + { + Id, + Name: Type.Literal("/nemoclaw-vllm"), + Image: ImageId, + StartedAt: Type.String({ minLength: 1, maxLength: 64 }), + Matches: Type.Literal(true), + State: Type.Object({ Running: Type.Literal(true) }), + Config: Type.Object({ + Env: Type.Array(Type.String({ pattern: "^VLLM_API_KEY=[a-f0-9]{64}$" }), { + minItems: 1, + maxItems: 1, + }), + Labels: Type.Record(Type.String(), Type.String({ maxLength: 512 })), + }), + NetworkSettings: Type.Object({ + Ports: Type.Object( + { + "8000/tcp": Type.Array(Type.Object({ HostIp: Text, HostPort: Text }), { + minItems: 2, + maxItems: 2, + }), + }, + { additionalProperties: false }, + ), + }), + }, + { additionalProperties: false }, +); + +export interface VllmExportRuntimeOptions { + readonly capture?: typeof dockerCapture; + readonly platform?: string; + readonly architecture?: string; + readonly homeDirectory?: string; + /** Test seam; production authentication remains inside the existing lifecycle owner. */ + readonly authentication?: Pick; +} + +function fail(): never { + throw new Error("The fixed managed vLLM runtime could not be verified for export."); +} + +function parse( + source: string, + schema: T, +): TypeBoxModule.Type.Static { + if (!source || Buffer.byteLength(source) > MAX_INSPECTION_BYTES) fail(); + const value: unknown = JSON.parse(source); + if (!Check(schema, value)) fail(); + return value; +} + +function equalJson(expression: string, expected: unknown): string { + // Go quoted strings contain only validated catalog/image data, never credentials. + return `(eq (json ${expression}) ${JSON.stringify(JSON.stringify(expected))})`; +} + +function emptyArray(expression: string): string { + return `(or ${equalJson(expression, null)} ${equalJson(expression, [])})`; +} + +function expectedRuntime(recorded: ServingProfileProvenance) { + const catalog = loadServingCatalog(); + const current = assertServingProfileProvenanceCurrent(recorded, catalog); + const recipe = loadManagedInferenceCatalog().recipes.find( + ({ metadata }) => metadata.id === EXPORTED_VLLM_RECIPE_ID, + ); + const imageRef = current.runtimeImage; + if ( + current.preset.id !== EXPORTED_VLLM_PROFILE_ID || + current.recipe.id !== EXPORTED_VLLM_RECIPE_ID || + !recipe || + !isHostLocalInferenceServingRecipe(recipe) || + recipe.spec.runtime.architecture !== "amd64" || + !recipe.spec.serve.directInstall || + recipe.spec.serve.directInstall.authentication !== "bearer" || + !recipe.spec.serve.directInstall.fixedArguments || + !recipe.spec.serve.directInstall.catalogReceipt || + recipe.spec.runtime.temporaryFilesystems.length !== 0 || + recipe.spec.runtime.devices.length !== 0 || + recipe.spec.runtime.gpuRequest !== "all" || + !isImmutableImageReference(imageRef) + ) + fail(); + const model = materializeHostLocalVllmModel(recipe, recipe.spec.serve.directInstall, "linux"); + if (model.maxModelLen !== EXPORTED_VLLM_CONTEXT_WINDOW) fail(); + return { current, recipe, imageRef, command: buildVllmServeCommand(model, {}) }; +} + +function containerFormat( + expected: ReturnType, + image: TypeBoxModule.Type.Static, + homeDirectory: string, +): string { + const { runtime } = expected.recipe.spec; + // Compare image defaults in Docker. A modified container environment never leaves the daemon, + // except for the single managed key consumed by the existing private authentication verifier. + const environmentCheck = image.Environment.map( + (value) => + `{{$found := false}}{{range .Config.Env}}{{if eq . ${JSON.stringify(value)}}}{{$found = true}}{{end}}{{end}}{{if not $found}}{{$environment = false}}{{end}}`, + ).join(""); + const labels = [ + HOST_LOCAL_VLLM_AUTH_LABEL, + HOST_LOCAL_VLLM_CATALOG_LABEL, + HOST_LOCAL_VLLM_MANAGED_LABEL, + HOST_LOCAL_VLLM_PRESET_DIGEST_LABEL, + HOST_LOCAL_VLLM_PRESET_LABEL, + HOST_LOCAL_VLLM_RECIPE_DIGEST_LABEL, + HOST_LOCAL_VLLM_RECIPE_LABEL, + "com.nvidia.nemoclaw.vllm-role", + ] + .map( + (key) => + `${JSON.stringify(key)}:{{json (or (index .Config.Labels ${JSON.stringify(key)}) "")}}`, + ) + .join(","); + const ulimitCheck = [ + ["memlock", runtime.ulimits.memlock === "unlimited" ? -1 : runtime.ulimits.memlock], + ["stack", runtime.ulimits.stackBytes], + ] + .map( + ([name, value]) => + `{{$found := false}}{{range .HostConfig.Ulimits}}{{if and (eq .Name ${JSON.stringify(name)}) ${equalJson(".Hard", value)} ${equalJson(".Soft", value)}}}{{$found = true}}{{end}}{{end}}{{if not $found}}{{$ulimits = false}}{{end}}`, + ) + .join(""); + const conditions = [ + equalJson(".Config.Cmd", ["-lc", expected.command]), + equalJson(".Config.Entrypoint", ["/bin/bash"]), + `(eq .Config.Image ${JSON.stringify(expected.current.runtimeImage)})`, + `(eq .Image ${JSON.stringify(image.Id)})`, + `(eq .HostConfig.NetworkMode "bridge")`, + `(eq .HostConfig.IpcMode ${JSON.stringify(runtime.ipcMode)})`, + equalJson(".HostConfig.ShmSize", runtime.sharedMemoryBytes), + `(eq .HostConfig.RestartPolicy.Name "unless-stopped")`, + `.HostConfig.Init`, + `(not .HostConfig.Privileged)`, + emptyArray(".HostConfig.Devices"), + emptyArray(".HostConfig.CapAdd"), + emptyArray(".HostConfig.SecurityOpt"), + `(eq (len .HostConfig.Ulimits) 2)`, + "$ulimits", + `(or ${equalJson(".HostConfig.Tmpfs", null)} ${equalJson(".HostConfig.Tmpfs", {})})`, + equalJson(".HostConfig.Memory", 0), + equalJson(".HostConfig.NanoCpus", 0), + `(eq (len .HostConfig.DeviceRequests) 1)`, + equalJson("(index .HostConfig.DeviceRequests 0).Count", -1), + emptyArray("(index .HostConfig.DeviceRequests 0).DeviceIDs"), + equalJson("(index .HostConfig.DeviceRequests 0).Capabilities", [["gpu"]]), + `(eq (len .Mounts) 1)`, + `(eq (index .Mounts 0).Type "bind")`, + `(eq (index .Mounts 0).Source ${JSON.stringify(path.join(homeDirectory, ".cache/huggingface/hub"))})`, + `(eq (index .Mounts 0).Destination ${JSON.stringify(`${runtime.modelCache.target}/hub`)})`, + `(not (index .Mounts 0).RW)`, + `(eq (len .Config.Env) ${String(image.Environment.length + 1)})`, + "$environment", + ]; + return `{{$environment := true}}{{$ulimits := true}}${environmentCheck}${ulimitCheck}{"Id":{{json .Id}},"Name":{{json .Name}},"Image":{{json .Image}},"StartedAt":{{json .State.StartedAt}},"Matches":{{and ${conditions.join(" ")}}},"State":{"Running":{{json .State.Running}}},"Config":{"Labels":{${labels}},"Env":[{{range .Config.Env}}{{if eq (index (split . "=") 0) "VLLM_API_KEY"}}{{json .}}{{end}}{{end}}]},"NetworkSettings":{"Ports":{{json .NetworkSettings.Ports}}}}`; +} + +/** Read the fixed runtime; authentication stays inside existing private lifecycle verification. */ +export function observeManagedVllmForExport( + recorded: ServingProfileProvenance, + options: VllmExportRuntimeOptions = {}, +): ObservedManagedVllmRuntime { + try { + if ( + (options.platform ?? process.platform) !== "linux" || + (options.architecture ?? process.arch) !== "x64" + ) + fail(); + const expected = expectedRuntime(recorded); + const capture = options.capture ?? dockerCapture; + const env = buildLocalManagedVllmDockerEnv(); + const inspect = (kind: string, name: string, format: string) => + capture([kind, "inspect", "--format", format, name], { + env, + timeout: INSPECTION_TIMEOUT_MS, + maxBuffer: MAX_INSPECTION_BYTES, + }); + const image = parse( + inspect( + "image", + expected.imageRef, + '{"Id":{{json .Id}},"Os":{{json .Os}},"Architecture":{{json .Architecture}},"Environment":{{json .Config.Env}}}', + ), + ImageSchema, + ); + if ( + new Set(image.Environment).size !== image.Environment.length || + image.Environment.some((value) => value.startsWith("VLLM_API_KEY=")) + ) + fail(); + const network = parse( + inspect( + "network", + "openshell-docker", + '{"Id":{{json .Id}},"Name":{{json .Name}},"Driver":{{json .Driver}},"Config":{{json .IPAM.Config}}}', + ), + NetworkSchema, + ); + const bridge = validateManagedVllmBridgeHost(network.Config[0]!.Gateway); + const row = parse( + inspect( + "container", + HOST_LOCAL_VLLM_CONTAINER_NAME, + containerFormat(expected, image, options.homeDirectory ?? os.homedir()), + ), + ContainerSchema, + ); + const identity = { + [HOST_LOCAL_VLLM_CATALOG_LABEL]: expected.current.catalogDigest, + [HOST_LOCAL_VLLM_PRESET_LABEL]: expected.current.preset.id, + [HOST_LOCAL_VLLM_PRESET_DIGEST_LABEL]: expected.current.preset.digest, + [HOST_LOCAL_VLLM_RECIPE_LABEL]: expected.current.recipe.id, + [HOST_LOCAL_VLLM_RECIPE_DIGEST_LABEL]: expected.current.recipe.digest, + }; + if (Object.entries(identity).some(([key, value]) => row.Config.Labels[key] !== value)) fail(); + const recovered = recoverHostLocalManagedVllmEndpoint({ + ...options.authentication, + dockerInspect: () => JSON.stringify([row]), + resolveBridgeHost: () => bridge, + }); + if (!recovered || recovered.containerId !== row.Id || row.Image !== image.Id) fail(); + const hostPort = Number(new URL(recovered.baseUrl).port); + const serving: NemoClawManagedVllmServing = { + backend: "vllm", + catalogDigest: expected.current.catalogDigest, + profile: { id: EXPORTED_VLLM_PROFILE_ID, digest: expected.current.preset.digest }, + recipe: { id: EXPORTED_VLLM_RECIPE_ID, digest: expected.current.recipe.digest }, + model: { ...expected.current.model, servedName: expected.recipe.spec.model.servedName }, + runtime: { image: { ref: expected.imageRef } }, + hostPort, + }; + if ( + !isDeepStrictEqual(expected.current.model, { + id: expected.recipe.spec.model.id, + revision: expected.recipe.spec.model.revision, + }) + ) + fail(); + return { + serving, + containerId: row.Id, + imageId: row.Image, + networkId: network.Id, + startedAt: row.StartedAt, + }; + } catch { + fail(); + } +} diff --git a/src/lib/inference/serving/vllm-host-local-lifecycle.test.ts b/src/lib/inference/serving/vllm-host-local-lifecycle.test.ts index d59d04fda35..f2564f9f215 100644 --- a/src/lib/inference/serving/vllm-host-local-lifecycle.test.ts +++ b/src/lib/inference/serving/vllm-host-local-lifecycle.test.ts @@ -17,6 +17,7 @@ import { HOST_LOCAL_VLLM_PRESET_LABEL, HOST_LOCAL_VLLM_RECIPE_DIGEST_LABEL, HOST_LOCAL_VLLM_RECIPE_LABEL, + HOST_LOCAL_VLLM_RUNTIME_RECEIPT_FILE, persistHostLocalVllmRuntimeReceipt, type RecoverHostLocalManagedVllmOptions, recoverHostLocalManagedVllmEndpoint, @@ -244,6 +245,26 @@ describe("host-local managed vLLM recovery", () => { ).toThrow("does not match its ownership receipt"); }); + it.each([ + ["undersized", "{"], + ["oversized", "x".repeat(64 * 1024 + 1)], + ])("rejects an %s owner-only receipt before loading credentials", (_kind, receipt) => { + const directory = stateDir(); + const receiptPath = path.join(directory, HOST_LOCAL_VLLM_RUNTIME_RECEIPT_FILE); + fs.writeFileSync(receiptPath, receipt, { mode: 0o600 }); + fs.chmodSync(receiptPath, 0o600); + const loadApiKey = vi.fn(() => API_KEY); + + expect(() => + recoverHostLocalManagedVllmEndpoint({ + dockerInspect: () => inspect(API_KEY, runtimeAuthFingerprint(API_KEY), PROFILE_LABELS), + loadApiKey, + stateDir: directory, + }), + ).toThrow("runtime receipt has an unexpected size"); + expect(loadApiKey).not.toHaveBeenCalled(); + }); + it("does not adopt a dual-Station container when every host-local marker also matches", () => { const observed = vi.fn(); expect( diff --git a/src/lib/inference/serving/vllm-host-local-lifecycle.ts b/src/lib/inference/serving/vllm-host-local-lifecycle.ts index 8aefd99ac3b..813d180df47 100644 --- a/src/lib/inference/serving/vllm-host-local-lifecycle.ts +++ b/src/lib/inference/serving/vllm-host-local-lifecycle.ts @@ -131,6 +131,9 @@ function readRuntimeReceipt(stateDir: string): unknown { ) { throw new Error("Managed host-local vLLM runtime receipt is not owner-only."); } + if (stat.size < 2 || stat.size > 64 * 1024) { + throw new Error("Managed host-local vLLM runtime receipt has an unexpected size."); + } try { return JSON.parse(fs.readFileSync(fd, "utf8")); } catch { diff --git a/test/e2e/README.md b/test/e2e/README.md index ec333122f06..6c638c00e67 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -462,6 +462,23 @@ The `gpu-double-onboard`, `gpu-e2e`, and `llama-cpp-generic-gpu` targets keep th Retained workflow jobs are exceptions to the catalogue shape. Keep one only for a multi-job handoff, an unrepresented credential boundary, or an execution contract the reusable profile cannot represent. +The `brave-search` target qualifies configuration export after normal Brave-enabled OpenClaw onboarding. +It validates two exports through the public schema, compares their specs, and requires a `BRAVE_API_KEY` reference without credential values or internal transports. +The target retains checks of the materialized OpenClaw search configuration, credential isolation, real agent search, direct Brave API results, and disabled-search reuse. +Private YAML files are removed during cleanup; artifacts retain redacted command results and an allowlisted qualification summary. +The export assertions replace redundant checks within the same Brave lifecycle. +Live policy qualification and a real Brave response cover the initial policy command and hostname substring. +Successful agent execution and its answer cover the negative diagnostic-text check. +Retained sandbox identity, materialized configuration, and HTTP egress cover the reused status command. +Complete JSON parsing and expected configuration fields cover config-read exit codes; valid exact UUID continuity covers sandbox-read exit codes. +The retained nonzero HTTP response covers the extra egress command exit check. +The lower direct assertion count is recorded in the census; transitive coverage remains unchanged. + +For manual PR qualification, select `jobs=brave-search` with Docker and leave `targets` empty. +Confirm that the target executes: an unavailable optional Brave credential can remove it from the plan. +Trusted `main` controls the 45-minute job limit. +Changes to `brave-search-helpers.ts` select the target through its catalogue ownership metadata. + ### Catalogue Execution Evidence Every catalogue execution writes `evidence-manifest.json` in its target artifact directory. diff --git a/test/e2e/fixtures/hermes-config-export-live.ts b/test/e2e/fixtures/hermes-config-export-live.ts index 8db4c11324b..13ec81def93 100644 --- a/test/e2e/fixtures/hermes-config-export-live.ts +++ b/test/e2e/fixtures/hermes-config-export-live.ts @@ -141,6 +141,8 @@ export async function verifyHermesConfigExportLive( const nemoclawDocument = validateNemoClawConfig(YAML.parse(nemoclawRaw)); const nemohermesDocument = validateNemoClawConfig(YAML.parse(nemohermesRaw)); const sandbox = nemoclawDocument.spec.sandboxes[0]!; + const provider = nemoclawDocument.spec.inferenceProviders[0]; + const hostedProvider = provider && !("serving" in provider) ? provider : undefined; const expectedPolicy = policy.ok ? YAML.parse(policy.value.document) : null; const expectedImage = entry.workload?.kind === "managed-image" ? entry.workload.reference : null; @@ -191,8 +193,7 @@ export async function verifyHermesConfigExportLive( agent: sandbox.agents[0]?.type, checked: true, credentialValuesOmitted: !containsCredential, - credentialReferenceMatches: - nemoclawDocument.spec.inferenceProviders[0]?.credential?.env === entry.credentialEnv, + credentialReferenceMatches: hostedProvider?.credential?.env === entry.credentialEnv, identityDriftPreventedPublication: typeof nemoclawDriftExitCode === "number" && nemoclawDriftExitCode !== 0 && @@ -204,8 +205,7 @@ export async function verifyHermesConfigExportLive( nemoclawDriftDiagnostics.includes("drifted") && nemohermesDriftDiagnostics.includes("drifted"), immutableManagedImageMatches: sandbox.runtime.image.ref === expectedImage, - inferenceEndpointMatches: - nemoclawDocument.spec.inferenceProviders[0]?.endpoint === entry.endpointUrl, + inferenceEndpointMatches: hostedProvider?.endpoint === entry.endpointUrl, launchersSucceeded, policyMatches: isDeepStrictEqual(sandbox.network.policy.explicit, expectedPolicy), sandboxNameMatches: sandbox.name === input.sandboxName, diff --git a/test/e2e/live/brave-search-helpers.ts b/test/e2e/live/brave-search-helpers.ts index 97854b0111c..c9c1e2932c1 100644 --- a/test/e2e/live/brave-search-helpers.ts +++ b/test/e2e/live/brave-search-helpers.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import YAML from "yaml"; +import { validateNemoClawConfig } from "../../../src/lib/config/schema.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { resultText } from "../fixtures/clients/index.ts"; @@ -167,6 +169,32 @@ export async function reuseBraveSandboxWithWebSearchDisabled( ); } +export async function exportBraveConfig( + host: HostCliClient, + outputPath: string, + artifactName: string, + redactionValues: string[], +): Promise { + return await host.command( + "node", + [CLI_ENTRYPOINT, "config", "export", SANDBOX_NAME, "--output", outputPath, "--json"], + { artifactName, cwd: REPO_ROOT, env: commandEnv(), redactionValues, timeoutMs: 60_000 }, + ); +} + +/** Validate private export output before retaining only public spec evidence. */ +export function assertBraveExport(raw: string, credentialValues: readonly string[]) { + for (const value of credentialValues) { + expect(raw.includes(value), "Export must omit credential values").toBe(false); + } + const document = validateNemoClawConfig(YAML.parse(raw)); + const webSearch = document.spec.sandboxes[0]?.integrations?.webSearch; + expect(webSearch?.provider).toBe("brave"); + expect(webSearch?.agentRefs).toEqual(["primary"]); + expect(webSearch?.credential.env).toBe("BRAVE_API_KEY"); + return document.spec; +} + export function assertBraveConfig(configText: string): string { const parsedConfig = JSON.parse(configText) as { tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; @@ -280,11 +308,3 @@ esac`, ); expect(probe.exitCode, "BRAVE_API_KEY is raw in the sandbox login shell environment").toBe(0); } - -export function assertBraveResponse(body: string): void { - const status = body.match(/HTTP_STATUS:(\d{3})/)?.[1]; - expect(status, body).toBe("200"); - const json = body.replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); - const braveResponse = JSON.parse(json) as { web?: { results?: unknown[] } }; - expect(braveResponse.web?.results?.length ?? 0, json.slice(0, 500)).toBeGreaterThan(0); -} diff --git a/test/e2e/live/brave-search.test.ts b/test/e2e/live/brave-search.test.ts index de89709d337..c76e2a2492f 100644 --- a/test/e2e/live/brave-search.test.ts +++ b/test/e2e/live/brave-search.test.ts @@ -1,19 +1,24 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { validateNemoClawConfig } from "../../../src/lib/config/schema.ts"; import { parseOpenShellSandboxId } from "../../../src/lib/adapters/openshell/sandbox-identity.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { parseOpenClawAgentText } from "../fixtures/openclaw-agent-output.ts"; -import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import { testTimeout } from "../../helpers/timeouts.ts"; import { assertBraveConfig, - assertBraveResponse, + assertBraveExport, assertBraveShellCredentialBoundary, cleanupBraveNemoClawSandbox, cleanupBraveState, commandEnv, + exportBraveConfig, onboardBrave, reuseBraveSandboxWithWebSearchDisabled, runBraveAgentWithSecretBoundaryCheck, @@ -24,14 +29,14 @@ import { const LIVE_TIMEOUT_MS = testTimeout(35 * 60_000); test( - "Brave search preset wires policy/config, performs real searches, and survives disabled-search reuse (#2687, #10404)", + "Brave search exports stable configuration, performs real searches, and survives disabled-search reuse (#2687, #10404, #10904)", { timeout: LIVE_TIMEOUT_MS, meta: { e2ePhases: [ "check Brave search prerequisites", "onboard Brave-enabled OpenClaw sandbox", - "validate Brave policy and secret isolation", + "export stable Brave configuration and verify secret isolation", "run Brave-backed OpenClaw search", "assert sandbox shell cannot read the real Brave key", "query Brave API through credential resolver", @@ -48,11 +53,12 @@ test( await artifacts.target.declare({ id: "brave-search", boundary: - "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", + "source CLI onboard/export + live SDK provider profile + in-sandbox OpenClaw/Brave API calls", sandboxName: SANDBOX_NAME, contracts: [ "onboard succeeds with BRAVE_API_KEY present", - "the brave network policy preset includes api.search.brave.com", + "config export validates the live managed Brave profile and produces schema-valid configuration", + "repeated export preserves the same spec and references BRAVE_API_KEY without credential values or internal transports", "OpenClaw web search config is enabled and selects provider=brave", "OpenClaw stores a BRAVE_API_KEY placeholder rather than the raw key", "OpenClaw agent can perform a Brave-backed web search", @@ -85,14 +91,50 @@ test( const onboard = await onboardBrave(host, braveKey, inferenceKey); expect(onboard.exitCode, resultText(onboard)).toBe(0); - progress.phase("validate Brave policy and secret isolation"); - const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { - artifactName: "phase-2-brave-policy", - env: commandEnv(), - timeoutMs: 60_000, + progress.phase("export stable Brave configuration and verify secret isolation"); + const exportDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brave-export-")); + cleanup.trackDisposable("remove private Brave config exports", () => + fs.rmSync(exportDirectory, { recursive: true, force: true }), + ); + const firstPath = path.join(exportDirectory, "first.yaml"); + const first = await exportBraveConfig( + host, + firstPath, + "phase-2-brave-config-export-first", + redactionValues, + ); + expect(first.exitCode, resultText(first)).toBe(0); + const firstRaw = fs.readFileSync(firstPath, "utf8"); + const firstSpec = assertBraveExport(firstRaw, redactionValues); + + const repeatPath = path.join(exportDirectory, "repeat.yaml"); + const repeat = await exportBraveConfig( + host, + repeatPath, + "phase-2-brave-config-export-repeat", + redactionValues, + ); + expect(repeat.exitCode, resultText(repeat)).toBe(0); + const repeatRaw = fs.readFileSync(repeatPath, "utf8"); + expect( + redactionValues.some((value) => repeatRaw.includes(value)), + "Repeated export must omit credential values", + ).toBe(false); + const repeatSpec = validateNemoClawConfig(YAML.parse(repeatRaw)).spec; + expect( + /NEMOCLAW_[A-Z0-9_]+|openshell:resolve:env:/u.test(firstRaw + repeatRaw), + "Export must omit internal environment transports and credential placeholders", + ).toBe(false); + expect(repeatSpec).toEqual(firstSpec); + await artifacts.writeJson("brave-config-export-evidence.json", { + sandboxName: SANDBOX_NAME, + provider: "brave", + credentialReference: "BRAVE_API_KEY", + schemaValid: true, + repeatedSpecMatches: true, + credentialValuesAbsent: true, + internalTransportsAbsent: true, }); - expect(policy.exitCode, resultText(policy)).toBe(0); - expect(resultText(policy)).toContain("api.search.brave.com"); const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { artifactName: "phase-2-openclaw-config", @@ -100,15 +142,11 @@ test( redactionValues, timeoutMs: 60_000, }); - expect(config.exitCode, resultText(config)).toBe(0); const placeholder = assertBraveConfig(config.stdout); progress.phase("run Brave-backed OpenClaw search"); const agent = await runBraveAgentWithSecretBoundaryCheck(sandbox, redactionValues); - expect(resultText(agent)).not.toMatch( - /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, - ); expect(agent.exitCode, resultText(agent)).toBe(0); expect(parseOpenClawAgentText(agent.stdout), resultText(agent)).toMatch( /nvidia|geforce|cuda|gpu/i, @@ -129,14 +167,17 @@ test( `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, { artifactName: "phase-4b-direct-brave-curl", timeoutMs: 60_000, redactionValues }, ); - assertBraveResponse(resultText(curl)); + const body = resultText(curl); + expect(body.match(/HTTP_STATUS:(\d{3})/)?.[1], body).toBe("200"); + const json = body.replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); + const braveResponse = JSON.parse(json) as { web?: { results?: unknown[] } }; + expect(braveResponse.web?.results?.length ?? 0, json.slice(0, 500)).toBeGreaterThan(0); progress.phase("re-onboard the existing sandbox with web search disabled"); const sandboxBeforeReuse = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { artifactName: "phase-5-pre-reuse-sandbox-identity", env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), timeoutMs: 60_000, }); - expect(sandboxBeforeReuse.exitCode, resultText(sandboxBeforeReuse)).toBe(0); const sandboxIdBeforeReuse = parseOpenShellSandboxId(resultText(sandboxBeforeReuse)); expect(sandboxIdBeforeReuse, resultText(sandboxBeforeReuse)).not.toBeNull(); @@ -149,20 +190,11 @@ test( env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), timeoutMs: 60_000, }); - expect(sandboxAfterReuse.exitCode, resultText(sandboxAfterReuse)).toBe(0); expect( parseOpenShellSandboxId(resultText(sandboxAfterReuse)), resultText(sandboxAfterReuse), ).toBe(sandboxIdBeforeReuse); - const status = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { - artifactName: "phase-6-reused-runtime-status", - cwd: REPO_ROOT, - env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), - timeoutMs: 60_000, - }); - expect(status.exitCode, resultText(status)).toBe(0); - const reusedConfig = await sandbox.exec( SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], @@ -172,7 +204,6 @@ test( timeoutMs: 60_000, }, ); - expect(reusedConfig.exitCode, resultText(reusedConfig)).toBe(0); const parsedReusedConfig = JSON.parse(reusedConfig.stdout) as { tools?: { web?: { search?: { enabled?: unknown } } }; }; @@ -191,7 +222,6 @@ test( "curl -sS -o /dev/null --max-time 20 -w 'HTTP_STATUS:%{http_code}\\n' 'https://api.search.brave.com/res/v1/web/search'", { artifactName: "phase-6-reused-brave-egress", timeoutMs: 60_000 }, ); - expect(reachable.exitCode, resultText(reachable)).toBe(0); expect(resultText(reachable)).toMatch(/HTTP_STATUS:(?!000)[0-9]{3}/u); }, ); diff --git a/test/e2e/live/network-policy.test.ts b/test/e2e/live/network-policy.test.ts index 82e077a1703..e22751cbc0e 100644 --- a/test/e2e/live/network-policy.test.ts +++ b/test/e2e/live/network-policy.test.ts @@ -364,9 +364,9 @@ test( expect(document.spec.sandboxes[0].runtime.image.ref).toBe( entry.workload?.kind === "managed-image" ? entry.workload.reference : null, ); - expect(document.spec.inferenceProviders[0].endpoint).toBe( - requireHostedInferenceConfig(secrets).endpointUrl, - ); + const exportedProvider = document.spec.inferenceProviders[0]; + const exportedEndpoint = "endpoint" in exportedProvider ? exportedProvider.endpoint : undefined; + expect(exportedEndpoint).toBe(requireHostedInferenceConfig(secrets).endpointUrl); expect(document.spec.sandboxes[0].network.policy.explicit).toEqual( policy.ok ? YAML.parse(policy.value.document) : null, ); @@ -397,7 +397,7 @@ test( await artifacts.writeJson("config-export-live-evidence.json", { sandboxName: SANDBOX_NAME, image: document.spec.sandboxes[0].runtime.image.ref, - endpoint: document.spec.inferenceProviders[0].endpoint, + endpoint: exportedEndpoint, effectivePolicyMatches: true, identityDriftPreventedPublication: true, }); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 3572561f894..6960dcfeccb 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -395,6 +395,8 @@ "live": "test/e2e/live/brave-search.test.ts", "liveSources": ["test/e2e/live/brave-search-helpers.ts"], "fast": [ + "src/lib/adapters/config/live-export-source.test.ts", + "src/lib/config/config.test.ts", "src/lib/adapters/openshell/sandbox-identity.test.ts", "src/lib/actions/sandbox/rebuild-backup-phase.test.ts", "src/lib/onboard/machine/handlers/agent-setup.test.ts", diff --git a/test/e2e/support/brave-search-config.test.ts b/test/e2e/support/brave-search-config.test.ts index 0aefbb128ac..bdf6b0f0a10 100644 --- a/test/e2e/support/brave-search-config.test.ts +++ b/test/e2e/support/brave-search-config.test.ts @@ -1,9 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import YAML from "yaml"; +import { buildExportConfig } from "../../../src/lib/domain/config/export-document.ts"; +import type { VerifiedExportSource } from "../../../src/lib/domain/config/export-evidence.ts"; +import { + parseNemoClawConfigDocumentName, + parseNemoClawConfigDocumentUid, +} from "../../../src/lib/config/model.ts"; import { describe, expect, it } from "vitest"; -import { assertBraveConfig } from "../live/brave-search-helpers.ts"; +import { assertBraveConfig, assertBraveExport } from "../live/brave-search-helpers.ts"; const VERSIONED_PLACEHOLDER = "openshell:resolve:env:v12590243949725316565_BRAVE_API_KEY"; const UNVERSIONED_PLACEHOLDER = "openshell:resolve:env:BRAVE_API_KEY"; @@ -43,3 +50,75 @@ describe("Brave Search E2E configuration assertion", () => { ).toThrow(); }); }); + +function exportedBraveConfig() { + return buildExportConfig( + { + sandboxName: "alpha", + agent: "openclaw", + runtime: { provider: "docker", imageRef: "nvcr.io/nvidia/nemoclaw@sha256:" + "a".repeat(64) }, + gateway: { name: "nemoclaw", port: 8080 }, + inference: { + provider: "nvidia-prod", + model: "test-model", + api: "openai-completions", + endpoint: "https://integrate.api.nvidia.com/v1", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + }, + webSearch: { + provider: "brave", + agentRefs: ["primary"], + credential: { env: "BRAVE_API_KEY" }, + }, + policy: { + version: 1, + network_policies: { + brave: { + name: "brave", + endpoints: [{ host: "api.search.brave.com", port: 443 }], + binaries: [{ path: "/usr/bin/node" }], + }, + }, + }, + } as unknown as VerifiedExportSource, + { + documentName: parseNemoClawConfigDocumentName("alpha"), + documentUid: parseNemoClawConfigDocumentUid("11111111-1111-4111-8111-111111111111"), + }, + ); +} + +describe("Brave Search E2E export assertion", () => { + it("returns the validated public spec without comparing generated document identity (#10904)", () => { + const document = exportedBraveConfig(); + expect(assertBraveExport(YAML.stringify(document), ["synthetic-secret"])).toEqual( + document.spec, + ); + }); + + it.each(["synthetic-brave-secret", "synthetic-inference-secret"])( + "rejects leaked %s before parsing without echoing it (#10904)", + (secret) => { + let error: unknown; + try { + assertBraveExport(`invalid: [${secret}`, [secret]); + } catch (caught) { + error = caught; + } + expect(error).toBeInstanceOf(Error); + expect(String(error)).not.toContain(secret); + }, + ); + + it("validates the whole document, including fields outside the integration (#10904)", () => { + const document = exportedBraveConfig(); + Object.assign(document, { unexpected: true }); + expect(() => assertBraveExport(YAML.stringify(document), [])).toThrow(); + }); + + it("requires the expected Brave integration even when the document schema allows its absence (#10904)", () => { + const document = exportedBraveConfig(); + Object.assign(document.spec.sandboxes[0]!, { integrations: undefined }); + expect(() => assertBraveExport(YAML.stringify(document), [])).toThrow(); + }); +}); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 4bcb0b323a3..85cd52ce33c 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -626,6 +626,16 @@ describe("E2E workflow plan", () => { expect(selectedWorkflowJobs(plan)).toEqual(["catalogue-standard", "jetson-nvmap-gpu"]); }); + it("selects Brave export qualification when its helper changes", () => { + const plan = buildE2eWorkflowPlan( + {}, + { changedFiles: ["test/e2e/live/brave-search-helpers.ts"] }, + ); + const rows = Object.values(plan.catalogueMatrices).flat(); + + expect(rows.map((row) => row.id)).toEqual(["brave-search"]); + }); + it("selects Hermes GPU startup when its output proof changes", () => { const plan = buildE2eWorkflowPlan( {}, diff --git a/test/fixtures/openshell-provider-profile.ts b/test/fixtures/openshell-provider-profile.ts new file mode 100644 index 00000000000..7a923814181 --- /dev/null +++ b/test/fixtures/openshell-provider-profile.ts @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** SDK response defaults for the managed Brave profile's non-secret boundary. */ +export function managedBraveProfile() { + return { + id: "brave", + source: "user", + scope: "workspace", + resourceVersion: 4n, + inferenceCapable: false, + credentials: [ + { + name: "api_key", + envVars: ["BRAVE_API_KEY"], + required: true, + authStyle: "header", + headerName: "x-subscription-token", + queryParam: "", + pathTemplate: "", + }, + ], + endpoints: [ + { + host: "api.search.brave.com", + port: 443, + ports: [], + protocol: "rest", + tls: "", + enforcement: "enforce", + access: "read-write", + rules: [], + allowedIps: [], + denyRules: [], + allowEncodedSlash: false, + persistedQueries: "", + graphqlPersistedQueries: {}, + graphqlMaxBodyBytes: 0, + path: "", + websocketCredentialRewrite: false, + requestBodyCredentialRewrite: false, + advisorProposed: false, + credentialSigning: "", + signingService: "", + signingRegion: "", + jsonRpcMaxBodyBytes: 0, + }, + ], + binaries: ["/usr/local/bin/node", "/usr/bin/node", "/usr/local/bin/curl", "/usr/bin/curl"].map( + (path) => ({ path }), + ), + }; +} diff --git a/test/onboarding/openshell-sdk-export-reads.test.ts b/test/onboarding/openshell-sdk-export-reads.test.ts index 90ff652f587..1f51a438b9c 100644 --- a/test/onboarding/openshell-sdk-export-reads.test.ts +++ b/test/onboarding/openshell-sdk-export-reads.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; import { inspect } from "node:util"; import { describe, expect, it, vi } from "vitest"; import YAML from "yaml"; @@ -25,6 +26,76 @@ function hasSdkArtifact(): boolean { } describe("released OpenShell SDK export reads", () => { + it.skipIf(!hasSdkArtifact()).each(["brave", "openai"] as const)( + "qualifies the checked-in %s profile through generated SDK responses (#10904)", + async (profileId) => { + const sdkPackage = "@nvidia/openshell-sdk/raw"; + const protobufPackage = "@bufbuild/protobuf"; + const [raw, { create, fromJson, toBinary, fromBinary }] = await Promise.all([ + import(sdkPackage), + import(protobufPackage), + ]); + const checkedIn = YAML.parse( + readFileSync( + new URL(`../../nemoclaw-blueprint/provider-profiles/${profileId}.yaml`, import.meta.url), + "utf8", + ), + ); + const profile = fromJson(raw.ProviderProfileSchema, { + id: checkedIn.id, + source: "user", + scope: "workspace", + resourceVersion: "4", + credentials: checkedIn.credentials, + endpoints: checkedIn.endpoints, + binaries: checkedIn.binaries.map((path: string) => ({ path })), + inference_capable: checkedIn.inference_capable, + }); + const roundTrip = (schema: unknown, input: unknown) => + fromBinary(schema, toBinary(schema, create(schema, input))); + const client: OpenShellReadClient = { + raw: { + getProvider: async () => + roundTrip(raw.OpenShell.method.getProvider.output, { + provider: { + metadata: { + id: "provider-id", + name: "alpha", + workspace: "default", + resourceVersion: 9n, + }, + type: profileId, + profileWorkspace: "default", + }, + }), + getProviderProfile: async () => + roundTrip(raw.OpenShell.method.getProviderProfile.output, { profile }), + getSandbox: async () => { + throw new Error("unexpected sandbox read"); + }, + getSandboxConfig: async () => { + throw new Error("unexpected config read"); + }, + }, + }; + const result = await createProviders(async () => client).get({ + target: { kind: "named", gatewayName: "nemoclaw" }, + workspace: "default", + name: "alpha", + configKeys: [], + profileContract: profileId, + signal: new AbortController().signal, + }); + expect(result?.managedProfile).toEqual({ + id: profileId, + source: "user", + scope: "workspace", + resourceVersion: "4", + }); + expect(result?.profileWorkspace).toBe("default"); + }, + ); + it.skipIf(!hasSdkArtifact()).each(["openai", "nvidia"] as const)( "accepts generated %s responses without losing identity or uint64 revisions", async (providerType) => { diff --git a/test/onboarding/vllm-export-docker-format.test.ts b/test/onboarding/vllm-export-docker-format.test.ts new file mode 100644 index 00000000000..1132e2bc4b5 --- /dev/null +++ b/test/onboarding/vllm-export-docker-format.test.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { dockerClientAvailable, startVllmExportFormatFixture } from "./vllm-export-format-fixture"; + +type Fixture = Awaited>; + +describe.skipIf(!dockerClientAvailable)("managed vLLM Docker format boundary", () => { + let fixture: Fixture; + beforeEach(async () => { + fixture = await startVllmExportFormatFixture(); + }); + afterEach(async () => { + await fixture?.close(); + }); + + it("renders the fixed runtime through the actual Docker client", () => { + expect(fixture.run().serving.hostPort).toBe(18000); + expect(JSON.stringify(fixture.calls)).not.toContain(fixture.key); + }); + + it("accepts reordered equivalent resource limits", () => { + fixture.objects.container.HostConfig.Ulimits.reverse(); + expect(fixture.run().serving.hostPort).toBe(18000); + }); + + it("accepts Docker null defaults for unused runtime settings", () => { + Object.assign(fixture.objects.container.HostConfig, { + Devices: null, + CapAdd: null, + SecurityOpt: null, + Tmpfs: null, + }); + Object.assign(fixture.objects.container.HostConfig.DeviceRequests[0]!, { DeviceIDs: null }); + expect(fixture.run().serving.hostPort).toBe(18000); + }); + + it.each([ + [ + "additional capability", + (f: Fixture) => Object.assign(f.objects.container.HostConfig, { CapAdd: ["SYS_ADMIN"] }), + ], + [ + "malformed optional settings", + (f: Fixture) => Object.assign(f.objects.container.HostConfig, { CapAdd: false }), + ], + ["command", (f: Fixture) => f.objects.container.Config.Cmd.push("--unrepresented")], + [ + "environment injection", + (f: Fixture) => f.objects.container.Config.Env.push('INJECTION={{printf "unsafe"}}'), + ], + [ + "entrypoint", + (f: Fixture) => { + f.objects.container.Config.Entrypoint = ["/bin/sh"]; + }, + ], + [ + "stopped container", + (f: Fixture) => { + f.objects.container.State.Running = false; + }, + ], + [ + "GPU request", + (f: Fixture) => { + f.objects.container.HostConfig.DeviceRequests = []; + }, + ], + [ + "resource limit", + (f: Fixture) => { + f.objects.container.HostConfig.Ulimits[0]!.Soft = 1; + }, + ], + [ + "writable mount", + (f: Fixture) => { + f.objects.container.Mounts[0]!.RW = true; + }, + ], + [ + "shared memory", + (f: Fixture) => { + f.objects.container.HostConfig.ShmSize = 1; + }, + ], + ])("rejects changed %s at the Docker format boundary", (_name, change) => { + change(fixture); + expect(fixture.run).toThrow("The fixed managed vLLM runtime could not be verified for export."); + }); + + it("keeps quoted image defaults literal and private authentication out of observations", () => { + const literal = 'LITERAL={{printf "unsafe"}}'; + fixture.objects.image.Config.Env.push(literal); + fixture.objects.container.Config.Env.push(literal); + const observation = JSON.stringify(fixture.run()); + expect(observation).not.toContain(fixture.key); + expect(observation).not.toContain(fixture.fingerprint); + expect(observation).not.toContain(fixture.directory); + expect(observation).not.toContain(literal); + expect(observation).not.toContain("VLLM_API_KEY"); + expect(JSON.stringify(fixture.calls)).not.toContain(fixture.key); + }); +}); diff --git a/test/onboarding/vllm-export-format-fixture.ts b/test/onboarding/vllm-export-format-fixture.ts new file mode 100644 index 00000000000..0f27a354583 --- /dev/null +++ b/test/onboarding/vllm-export-format-fixture.ts @@ -0,0 +1,203 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { Worker } from "node:worker_threads"; +import { + observeManagedVllmForExport, + type VllmExportRuntimeOptions, +} from "../../src/lib/inference/serving/vllm-export-runtime"; +import { EXPORTED_VLLM_PROFILE_ID, EXPORTED_VLLM_RECIPE_ID } from "../../src/lib/config/model"; +import { + loadManagedInferenceCatalog, + loadServingCatalog, +} from "../../src/lib/inference/serving/catalog-loader"; +import { isHostLocalInferenceServingRecipe } from "../../src/lib/inference/serving/adapter-registry"; +import { servingProfileProvenance } from "../../src/lib/inference/serving/profile-provenance"; +import { materializeHostLocalVllmModel } from "../../src/lib/inference/serving/host-local-vllm-selection"; +import { buildVllmServeCommand } from "../../src/lib/inference/vllm-models"; +import * as lifecycle from "../../src/lib/inference/serving/vllm-host-local-lifecycle"; +import { runtimeAuthFingerprint } from "../../src/lib/inference/serving/runtime-auth-fingerprint"; + +export function vllmExportFormatFixture(directory: string) { + const provenance = servingProfileProvenance(loadServingCatalog(), EXPORTED_VLLM_PROFILE_ID); + const recipe = loadManagedInferenceCatalog().recipes.find( + ({ metadata }) => metadata.id === EXPORTED_VLLM_RECIPE_ID, + ); + if (!recipe || !isHostLocalInferenceServingRecipe(recipe) || !recipe.spec.serve.directInstall) + throw new Error("Fixture requires the fixed host-local recipe"); + const model = materializeHostLocalVllmModel(recipe, recipe.spec.serve.directInstall, "linux"); + const { runtime } = recipe.spec; + const key = "e".repeat(64); + const fingerprint = runtimeAuthFingerprint(key); + const containerId = "a".repeat(64); + const imageId = `sha256:${"b".repeat(64)}`; + const labels = { + [lifecycle.HOST_LOCAL_VLLM_AUTH_LABEL]: fingerprint, + [lifecycle.HOST_LOCAL_VLLM_MANAGED_LABEL]: "true", + [lifecycle.HOST_LOCAL_VLLM_CATALOG_LABEL]: provenance.catalogDigest, + [lifecycle.HOST_LOCAL_VLLM_PRESET_LABEL]: provenance.preset.id, + [lifecycle.HOST_LOCAL_VLLM_PRESET_DIGEST_LABEL]: provenance.preset.digest, + [lifecycle.HOST_LOCAL_VLLM_RECIPE_LABEL]: provenance.recipe.id, + [lifecycle.HOST_LOCAL_VLLM_RECIPE_DIGEST_LABEL]: provenance.recipe.digest, + }; + lifecycle.persistHostLocalVllmRuntimeReceipt( + { + containerId, + authFingerprint: fingerprint, + serving: { + catalogDigest: provenance.catalogDigest, + presetId: provenance.preset.id, + presetDigest: provenance.preset.digest, + recipeId: provenance.recipe.id, + recipeDigest: provenance.recipe.digest, + }, + }, + directory, + ); + return { + provenance, + key, + fingerprint, + objects: { + image: { + Id: imageId, + Os: "linux", + Architecture: "amd64", + Config: { Env: ["PATH=/usr/bin"] }, + }, + network: { + Id: "c".repeat(64), + Name: "openshell-docker", + Driver: "bridge", + IPAM: { Config: [{ Gateway: "172.18.0.1", Subnet: "172.18.0.0/16" }] }, + }, + container: { + Id: containerId, + Name: "/nemoclaw-vllm", + Image: imageId, + State: { Running: true, StartedAt: "2026-09-10T10:00:00Z" }, + Config: { + Image: provenance.runtimeImage, + Cmd: ["-lc", buildVllmServeCommand(model, {})], + Entrypoint: ["/bin/bash"], + Env: ["PATH=/usr/bin", `VLLM_API_KEY=${key}`], + Labels: labels, + }, + HostConfig: { + NetworkMode: "bridge", + IpcMode: runtime.ipcMode, + ShmSize: runtime.sharedMemoryBytes, + RestartPolicy: { Name: "unless-stopped" }, + Init: true, + Privileged: false, + Devices: [], + CapAdd: [], + SecurityOpt: [], + Memory: 0, + NanoCpus: 0, + Tmpfs: {}, + Ulimits: [ + { Name: "memlock", Hard: -1, Soft: -1 }, + { Name: "stack", Hard: runtime.ulimits.stackBytes, Soft: runtime.ulimits.stackBytes }, + ], + DeviceRequests: [{ Count: -1, DeviceIDs: [], Capabilities: [["gpu"]] }], + }, + Mounts: [ + { + Type: "bind", + Source: "/home/fixture/.cache/huggingface/hub", + Destination: `${runtime.modelCache.target}/hub`, + RW: false, + }, + ], + NetworkSettings: { + Ports: { + "8000/tcp": [ + { HostIp: "127.0.0.1", HostPort: "18000" }, + { HostIp: "172.18.0.1", HostPort: "18000" }, + ], + }, + }, + }, + }, + }; +} + +export const dockerClientAvailable = + process.platform !== "win32" && + spawnSync("docker", ["--version"], { timeout: 5000, stdio: "ignore" }).status === 0; + +// A private fake API exercises the installed Docker client's real Go template renderer. +// No host Docker daemon, container, GPU, image pull, or external network is involved. +const daemonSource = ` +const { parentPort, workerData } = require("node:worker_threads"); +const http = require("node:http"); +const fs = require("node:fs"); +const server = http.createServer((request, response) => { + const route = decodeURIComponent(request.url).replace(/^\\/v[0-9.]+/, ""); + const objects = JSON.parse(fs.readFileSync(workerData.objects, "utf8")); + const result = route.startsWith("/images/") ? objects.image : route.startsWith("/networks/") ? objects.network : route.startsWith("/containers/") ? objects.container : null; + response.writeHead(result ? 200 : 404, { "Content-Type": "application/json" }); + response.end(JSON.stringify(result ?? { message: "Unexpected request" })); +}); +server.listen(workerData.socket, () => parentPort.postMessage("ready")); +`; + +export async function startVllmExportFormatFixture() { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nc-vllm-format-")); + const socket = path.join(directory, "docker.sock"); + const objectsPath = path.join(directory, "objects.json"); + const fixture = vllmExportFormatFixture(directory); + const daemon = new Worker(daemonSource, { + eval: true, + workerData: { socket, objects: objectsPath }, + }); + const close = async () => { + await daemon.terminate(); + fs.rmSync(directory, { recursive: true, force: true }); + }; + try { + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Fixture server deadline exceeded")), 5000); + daemon.once("error", (error) => { + clearTimeout(timer); + reject(error); + }); + daemon.once("message", () => { + clearTimeout(timer); + resolve(); + }); + }); + } catch (error) { + await close(); + throw error; + } + const calls: string[][] = []; + const capture: NonNullable = (args, options) => { + calls.push([...args]); + const env: NodeJS.ProcessEnv = { ...options?.env, DOCKER_API_VERSION: "1.52" }; + delete env.DOCKER_CONTEXT; + const result = spawnSync("docker", ["--host", `unix://${socket}`, ...args], { + ...options, + env, + encoding: "utf8", + }); + if (result.status !== 0) throw new Error("Docker format inspection failed"); + return result.stdout; + }; + const run = () => { + fs.writeFileSync(objectsPath, JSON.stringify(fixture.objects)); + return observeManagedVllmForExport(fixture.provenance, { + capture, + platform: "linux", + architecture: "x64", + homeDirectory: "/home/fixture", + authentication: { stateDir: directory, loadApiKey: () => fixture.key }, + }); + }; + return { ...fixture, directory, run, calls, close }; +} diff --git a/tools/e2e/target-catalogue.mts b/tools/e2e/target-catalogue.mts index d5f3d3f4bcc..1415b2ed885 100644 --- a/tools/e2e/target-catalogue.mts +++ b/tools/e2e/target-catalogue.mts @@ -483,6 +483,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ installNonInteractive: true, restoreCli: true, exposeCliBin: true, + owningPaths: ["test/e2e/live/brave-search-helpers.ts"], environment: { ...hostedInference, ...nonInteractive,