diff --git a/scripts/checks/no-unit-blocks-in-live-e2e.ts b/scripts/checks/no-unit-blocks-in-live-e2e.ts new file mode 100644 index 00000000000..5b9b86bffcd --- /dev/null +++ b/scripts/checks/no-unit-blocks-in-live-e2e.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Guard: pure unit blocks must not hide inside test/e2e/live/** files. +// +// vitest.config.ts only collects test/e2e/live/**/*.test.ts when live E2E is +// enabled (NEMOCLAW_RUN_LIVE_E2E=1). On PR CI that flag is false, so the entire +// file is uncollected — including any `describe(...)` unit block embedded in it. +// Such blocks are dead weight on PR CI: they read like coverage but never run +// where they could. This is exactly how two mockable regressions stayed +// unguarded (the skill-agent classifiers and the openclaw TUI-correlation +// logic, the latter saved only by a lucky root-level duplicate). +// +// Convention this guard enforces: inside test/e2e/live/**, the vitest unit +// primitive `it(` is banned. Live cases are declared with `test` — directly, or +// (more often) through a gate wrapper assigned from `shouldRunLiveE2E() ? test +// : test.skip` / `test.skipIf(!shouldRunLiveE2E())`, sometimes grouped under +// `describe.sequential(...)`. A live case never needs `it(`; when `it(` appears +// in a live file it is invariably a pure-unit block someone parked there (as +// happened with the skill-agent and messaging classifier blocks). Such a block +// is dead on PR CI and belongs in an importable module + a PR-collected test +// (root test/**, a co-located src/**/*.test.ts, or test/e2e/support/**). +// +// We deliberately do NOT try to flag bare `test(` unit cases: a live test that +// uses module-level helpers legitimately reads as `test("...", async () => …)` +// with no fixture, and is syntactically indistinguishable from a unit case. The +// `it(` ban is the reliable, zero-false-positive line. + +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const LIVE_DIR = path.join(REPO_ROOT, "test", "e2e", "live"); +const TEST_FILE_PATTERN = /\.(?:test|spec)\.(?:[cm]?[jt]s)$/; +// Match the vitest unit primitive `it(` — including `it.each(`, `it.only(`, +// `it.skip(`, etc. — as a call at a statement boundary. The leading boundary +// (line start or whitespace) prevents matching inside a custom identifier, and +// requiring a call paren after the optional member keeps non-call references +// from matching. +const IT_PRIMITIVE_PATTERN = + /(?:^|[\s;{(])it(?:\.(?:each|only|skip|todo|fails|concurrent|sequential))?\s*\(/; + +export type LiveUnitBlockViolation = { + readonly file: string; + readonly line: number; + readonly text: string; +}; + +function toRepoPath(absPath: string): string { + return path.relative(REPO_ROOT, absPath).split(path.sep).join("/"); +} + +function* walkFiles(dir: string): Generator { + if (!existsSync(dir)) return; + for (const entry of readdirSync(dir)) { + const absPath = path.join(dir, entry); + const stats = statSync(absPath); + if (stats.isDirectory()) { + yield* walkFiles(absPath); + } else if (stats.isFile() && TEST_FILE_PATTERN.test(entry)) { + yield absPath; + } + } +} + +export function findLiveUnitBlocks(source: string, file: string): LiveUnitBlockViolation[] { + const violations: LiveUnitBlockViolation[] = []; + const lines = source.split(/\r\n|\r|\n/); + for (let i = 0; i < lines.length; i += 1) { + const text = lines[i] ?? ""; + const trimmed = text.trimStart(); + // Skip import lines (`import { it, test } from "vitest"`) and comments. + if (trimmed.startsWith("import ") || trimmed.startsWith("//") || trimmed.startsWith("*")) { + continue; + } + if (IT_PRIMITIVE_PATTERN.test(text)) { + violations.push({ file, line: i + 1, text: trimmed }); + } + } + return violations; +} + +export function collectLiveUnitBlocks(dir = LIVE_DIR): LiveUnitBlockViolation[] { + return [...walkFiles(dir)] + .flatMap((absPath) => findLiveUnitBlocks(readFileSync(absPath, "utf-8"), toRepoPath(absPath))) + .sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line); +} + +export function formatViolations(violations: readonly LiveUnitBlockViolation[]): string { + const out = [ + "Live E2E unit-block guard failed.", + "", + "These test/e2e/live/** files use the vitest unit primitive it(...). That glob", + "is only collected when NEMOCLAW_RUN_LIVE_E2E=1, so an it(...) block never runs", + "on PR CI — it looks like coverage but guards nothing. Live cases use test(...)", + "(directly or via a gate wrapper); it(...) in a live file is always a pure-unit", + "block parked in the wrong place.", + "", + "Fix: extract the helper under test into an importable module (src/** or", + "test/e2e/support/**) and move the it(...) block to a PR-collected project", + "(root test/**/*.test.ts, a co-located src/**/*.test.ts, or test/e2e/support/**).", + "Keep the live test importing the shared helper.", + "", + ]; + for (const v of violations) { + out.push(`- ${v.file}:${v.line} ${v.text}`); + } + return out.join("\n"); +} + +function main(): void { + const violations = collectLiveUnitBlocks(); + if (violations.length > 0) { + console.error(formatViolations(violations)); + process.exitCode = 1; + return; + } + console.log("Live E2E unit-block guard passed: no it(...) blocks in test/e2e/live/**."); +} + +if (fileURLToPath(import.meta.url) === path.resolve(process.argv[1] ?? "")) { + main(); +} diff --git a/scripts/checks/run.ts b/scripts/checks/run.ts index f02cf224add..9248bb9041a 100644 --- a/scripts/checks/run.ts +++ b/scripts/checks/run.ts @@ -61,6 +61,11 @@ const CHECKS: readonly CheckCommand[] = [ command: TSX, args: ["scripts/checks/test-title-style.ts"], }, + { + name: "no-unit-blocks-in-live-e2e", + command: TSX, + args: ["scripts/checks/no-unit-blocks-in-live-e2e.ts"], + }, ]; function main(): void { diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index e246fc56c4e..f70da66df13 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -1907,6 +1907,9 @@ node_options_has_require() { local token local tokens=() IFS=$' \t\n' read -r -a tokens <<<"${NODE_OPTIONS:-}" + # Iterating "${tokens[@]}" on an empty array trips `set -u` on bash 3.2 + # (macOS default); guard so the local unit harnesses run there too. + [ "${#tokens[@]}" -gt 0 ] || return 1 for token in "${tokens[@]}"; do if [ "$previous" = "--require" ] && [ "$token" = "$wanted" ]; then return 0 @@ -2014,7 +2017,7 @@ validate_nemoclaw_tmp_permissions() { [ -n "$_target" ] && _dynamic_targets+=("$_target") done < <(messaging_runtime_preload_targets) - validate_tmp_permissions "$_SANDBOX_SAFETY_NET" "$_PROXY_FIX_SCRIPT" "$_NEMOTRON_FIX_SCRIPT" "$_WS_FIX_SCRIPT" "$_SECCOMP_GUARD_SCRIPT" "$_CIAO_GUARD_SCRIPT" "${_dynamic_targets[@]}" + validate_tmp_permissions "$_SANDBOX_SAFETY_NET" "$_PROXY_FIX_SCRIPT" "$_NEMOTRON_FIX_SCRIPT" "$_WS_FIX_SCRIPT" "$_SECCOMP_GUARD_SCRIPT" "$_CIAO_GUARD_SCRIPT" "${_dynamic_targets[@]+"${_dynamic_targets[@]}"}" } verify_messaging_runtime_secret_scans() { @@ -2374,7 +2377,7 @@ start_auto_pair() { if [ "$(id -u)" -eq 0 ]; then run_prefix=("${STEP_DOWN_PREFIX_SANDBOX[@]}") fi - OPENCLAW_BIN="$OPENCLAW" nohup "${run_prefix[@]}" python3 -u - <<'PYAUTOPAIR' >>/tmp/auto-pair.log 2>&1 & + OPENCLAW_BIN="$OPENCLAW" nohup "${run_prefix[@]+"${run_prefix[@]}"}" python3 -u - <<'PYAUTOPAIR' >>/tmp/auto-pair.log 2>&1 & import json import importlib.util import os diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 66cb5123f69..4634fb81e6d 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -750,6 +750,19 @@ describe("runSandboxSnapshot", () => { expect(output).toContain("2 snapshot(s). Restore with:"); }); + it("prints create, list, and restore usage for the bare help branch", async () => { + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "help" }); + + const output = consoleLog.mock.calls.flat().join("\n"); + expect(output).toContain("Usage:"); + expect(output).toContain("alpha snapshot create"); + expect(output).toContain("alpha snapshot list"); + expect(output).toContain("alpha snapshot restore"); + }); + it("restores the latest snapshot into the source sandbox", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); getLatestBackupMock.mockReturnValue({ diff --git a/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact-test-helpers.ts b/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact-test-helpers.ts new file mode 100644 index 00000000000..58d4278b718 --- /dev/null +++ b/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact-test-helpers.ts @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Test harness helpers for whatsapp-qr-compact.test.ts. The Module._load hook +// keeps the test body linear; the routing decision itself reuses the runtime's +// exported resolvePatchedModule so the test exercises real production logic +// rather than a re-implemented copy. + +import { resolvePatchedModule } from "./whatsapp-qr-compact"; + +/** + * Build a Module._load wrapper identical to the runtime's: for the given + * absolute path it returns `patchedModule`, otherwise a bare object, then + * delegates to the runtime's resolvePatchedModule so patching happens only for + * qrcode-shaped requests and never leaks onto passthrough modules. + */ +export function makeQrcodeLoadHook( + absolutePath: string, + patchedModule: unknown, +): (request: unknown, ...rest: unknown[]) => unknown { + return function (request: unknown, ..._rest: unknown[]) { + const loaded = request === absolutePath ? patchedModule : {}; + return resolvePatchedModule(request, loaded); + }; +} diff --git a/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact.test.ts b/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact.test.ts new file mode 100644 index 00000000000..7485bbf204e --- /dev/null +++ b/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact.test.ts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Unit coverage for the WhatsApp compact-QR preload's pure shape-detect and +// patch helpers (NemoClaw#4522 wrong-package-patch regression class). The live +// whatsapp-qr-compact E2E only asserts terminal row counts against the real +// upstream renderer; these tests pin the load-hook contract hermetically with +// fake module objects so no real qrcode / qrcode-terminal dependency is needed. + +import { describe, expect, it, vi } from "vitest"; + +import { + isQrcodePackage, + isQrcodeTerminalPackage, + patchQrcode, + patchQrcodeTerminal, +} from "./whatsapp-qr-compact"; +import { makeQrcodeLoadHook } from "./whatsapp-qr-compact-test-helpers"; + +// A fake of the `qrcode` package main: has its OWN toString + create(). +function makeQrcodeFake() { + const calls: Array<{ text: unknown; opts: unknown; cb: unknown }> = []; + const mod = { + calls, + create() { + return {}; + }, + toString(text: unknown, opts?: unknown, cb?: unknown) { + calls.push({ text, opts, cb }); + return "QR"; + }, + }; + return mod; +} + +// A fake of the `qrcode-terminal` package: has generate(), no create(). +function makeQrcodeTerminalFake() { + const calls: Array<{ text: unknown; opts: unknown; cb: unknown }> = []; + const mod = { + calls, + generate(text: unknown, opts?: unknown, cb?: unknown) { + calls.push({ text, opts, cb }); + }, + }; + return mod; +} + +describe("isQrcodePackage (#4522)", () => { + it("detects the qrcode package main by own toString + create", () => { + expect(isQrcodePackage(makeQrcodeFake())).toBe(true); + }); + + it("does not match a lookalike submodule that only has create()", () => { + // qrcode's internal lib/core/qrcode.js exposes create() but only the + // inherited Object.prototype.toString — it must NOT be patched. + const submodule = { + create() { + return {}; + }, + }; + expect(isQrcodePackage(submodule)).toBe(false); + }); + + it("does not match qrcode-terminal (has generate, no create)", () => { + expect(isQrcodePackage(makeQrcodeTerminalFake())).toBe(false); + }); +}); + +describe("isQrcodeTerminalPackage (#4522)", () => { + it("detects qrcode-terminal by own generate and absent create", () => { + expect(isQrcodeTerminalPackage(makeQrcodeTerminalFake())).toBe(true); + }); + + it("does not match the qrcode package (has create)", () => { + expect(isQrcodeTerminalPackage(makeQrcodeFake())).toBe(false); + }); +}); + +describe("patchQrcode (#4522)", () => { + it("forces small:true only for terminal renders", () => { + const mod = makeQrcodeFake(); + patchQrcode(mod); + mod.toString("payload", { type: "terminal" }); + expect(mod.calls[0].opts).toEqual({ type: "terminal", small: true }); + }); + + it.each(["svg", "png", "utf8"])("leaves type=%s options untouched", (type) => { + const mod = makeQrcodeFake(); + patchQrcode(mod); + mod.toString("payload", { type }); + expect(mod.calls[0].opts).toEqual({ type }); + expect((mod.calls[0].opts as Record).small).toBeUndefined(); + }); + + it("does not mutate the caller-supplied options object", () => { + const mod = makeQrcodeFake(); + patchQrcode(mod); + const opts = { type: "terminal" }; + mod.toString("payload", opts); + expect(opts).toEqual({ type: "terminal" }); + }); + + it("preserves the toString(text, cb) signature", () => { + const mod = makeQrcodeFake(); + patchQrcode(mod); + const cb = vi.fn(); + mod.toString("payload", cb); + expect(mod.calls[0].cb).toBe(cb); + // No opts object was supplied, so nothing is forced. + expect(mod.calls[0].opts).toEqual({}); + }); + + it("is idempotent: double-patch does not re-wrap", () => { + const mod = makeQrcodeFake(); + patchQrcode(mod); + const wrappedOnce = mod.toString; + patchQrcode(mod); + expect(mod.toString).toBe(wrappedOnce); + // And forcing still works exactly once. + mod.toString("payload", { type: "terminal" }); + expect(mod.calls[0].opts).toEqual({ type: "terminal", small: true }); + }); +}); + +describe("patchQrcodeTerminal (#4522)", () => { + it("forces small:true on generate", () => { + const mod = makeQrcodeTerminalFake(); + patchQrcodeTerminal(mod); + mod.generate("payload", {}); + expect(mod.calls[0].opts).toEqual({ small: true }); + }); + + it("is idempotent: double-patch does not re-wrap", () => { + const mod = makeQrcodeTerminalFake(); + patchQrcodeTerminal(mod); + const wrappedOnce = mod.generate; + patchQrcodeTerminal(mod); + expect(mod.generate).toBe(wrappedOnce); + }); +}); + +describe("Module._load hook path-segment matching (#4522)", () => { + it('patches import("qrcode")\'s resolved absolute path', async () => { + // Simulate the real load hook: install a Module._load wrapper identical to + // the runtime's, then require by an ABSOLUTE resolved path (as import() + // bottoms out at) and confirm the returned module got the compact patch. + const Module = (await import("node:module")).default as unknown as { + _load: (...args: unknown[]) => unknown; + }; + const qrcodeFake = makeQrcodeFake(); + const absolutePath = "/tmp/app/node_modules/qrcode/lib/index.js"; + const origLoad = Module._load; + Module._load = makeQrcodeLoadHook(absolutePath, qrcodeFake); + try { + const loaded = Module._load(absolutePath) as ReturnType; + expect(loaded).toBe(qrcodeFake); + loaded.toString("payload", { type: "terminal" }); + expect(loaded.calls[0].opts).toEqual({ type: "terminal", small: true }); + } finally { + Module._load = origLoad; + } + }); +}); diff --git a/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact.ts b/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact.ts index dd126249f44..7454ae569b4 100644 --- a/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact.ts +++ b/src/lib/messaging/channels/whatsapp/runtime/whatsapp-qr-compact.ts @@ -45,118 +45,143 @@ // // Ref: https://github.com/NVIDIA/NemoClaw/issues/4522 -(function () { - "use strict"; - - if (process.__nemoclawWhatsappQrCompactInstalled) return; +function markPatched(mod) { try { - Object.defineProperty(process, "__nemoclawWhatsappQrCompactInstalled", { value: true }); + Object.defineProperty(mod, "__nemoclawCompactPatched", { value: true }); } catch (_e) { - process.__nemoclawWhatsappQrCompactInstalled = true; + mod.__nemoclawCompactPatched = true; } +} - var Module = require("module"); - var origLoad = Module._load; +function hasOwn(mod, name) { + return mod && Object.prototype.hasOwnProperty.call(mod, name); +} - function markPatched(mod) { - try { - Object.defineProperty(mod, "__nemoclawCompactPatched", { value: true }); - } catch (_e) { - mod.__nemoclawCompactPatched = true; +// `qrcode` package main: renderQrTerminal() calls qrcode.toString(text, opts). +// Require an OWN toString (every object inherits Object.prototype.toString, so +// a plain `typeof mod.toString` check would also match qrcode's internal +// submodules — e.g. lib/core/qrcode.js, which exposes create() but only the +// inherited toString — and needlessly mutate them). The package main exposes +// its own toString + create; the submodules do not have an own toString. +function isQrcodePackage(mod) { + return ( + hasOwn(mod, "toString") && + typeof mod.toString === "function" && + typeof mod.create === "function" + ); +} + +// `qrcode-terminal` package: exposes its own generate(text, opts, cb) and, +// unlike `qrcode`, has no create(). +function isQrcodeTerminalPackage(mod) { + return ( + hasOwn(mod, "generate") && + typeof mod.generate === "function" && + typeof mod.create !== "function" + ); +} + +function patchQrcode(mod) { + if (mod.__nemoclawCompactPatched) return mod; + var origToString = mod.toString; + mod.toString = function (text, opts, cb) { + // Support toString(text, cb) and toString(text, opts, cb) / (text, opts). + if (typeof opts === "function") { + cb = opts; + opts = undefined; } - } + var merged = {}; + if (opts && typeof opts === "object") { + for (var key in opts) { + if (Object.prototype.hasOwnProperty.call(opts, key)) merged[key] = opts[key]; + } + } + // Only the terminal renderer has the oversize problem. `type` defaults + // to "utf8" in the qrcode package, but the WhatsApp path always passes + // "terminal" explicitly; force small there and leave every other type + // (svg/png/utf8 data URIs used elsewhere) exactly as the caller asked. + if (merged.type === "terminal") { + merged.small = true; + } + return origToString.call(this, text, merged, cb); + }; + markPatched(mod); + return mod; +} - function hasOwn(mod, name) { - return mod && Object.prototype.hasOwnProperty.call(mod, name); - } +function patchQrcodeTerminal(mod) { + if (mod.__nemoclawCompactPatched) return mod; + var origGenerate = mod.generate; + mod.generate = function (text, opts, cb) { + if (typeof opts === "function") { + cb = opts; + opts = undefined; + } + var merged = {}; + if (opts && typeof opts === "object") { + for (var key in opts) { + if (Object.prototype.hasOwnProperty.call(opts, key)) merged[key] = opts[key]; + } + } + merged.small = true; + return origGenerate.call(this, text, merged, cb); + }; + markPatched(mod); + return mod; +} - // `qrcode` package main: renderQrTerminal() calls qrcode.toString(text, opts). - // Require an OWN toString (every object inherits Object.prototype.toString, so - // a plain `typeof mod.toString` check would also match qrcode's internal - // submodules — e.g. lib/core/qrcode.js, which exposes create() but only the - // inherited toString — and needlessly mutate them). The package main exposes - // its own toString + create; the submodules do not have an own toString. - function isQrcodePackage(mod) { - return ( - hasOwn(mod, "toString") && - typeof mod.toString === "function" && - typeof mod.create === "function" - ); +// Pure routing decision shared by the installed hook and its tests. Only a +// request string that mentions qrcode is eligible; the shape-detect guards then +// decide which patch (if any) applies. Keeping the request filter ahead of the +// patch calls means a non-qrcode request never mutates `loaded` as a side +// effect. A patch failure degrades to the unpatched module. +function resolvePatchedModule(request, loaded) { + if (typeof request === "string" && request.indexOf("qrcode") !== -1) { + try { + if (isQrcodePackage(loaded)) return patchQrcode(loaded); + if (isQrcodeTerminalPackage(loaded)) return patchQrcodeTerminal(loaded); + } catch (_e) { + return loaded; + } } + return loaded; +} - // `qrcode-terminal` package: exposes its own generate(text, opts, cb) and, - // unlike `qrcode`, has no create(). - function isQrcodeTerminalPackage(mod) { - return ( - hasOwn(mod, "generate") && - typeof mod.generate === "function" && - typeof mod.create !== "function" - ); - } +// Named exports so the pure shape-detect + patch helpers can be unit-tested +// (NemoClaw#4522 regression class) without pulling in a real qrcode dependency. +// The auto-install below still uses the exact same functions, so the runtime +// hook behaves identically. +export { + hasOwn, + isQrcodePackage, + isQrcodeTerminalPackage, + patchQrcode, + patchQrcodeTerminal, + resolvePatchedModule, +}; - function patchQrcode(mod) { - if (mod.__nemoclawCompactPatched) return mod; - var origToString = mod.toString; - mod.toString = function (text, opts, cb) { - // Support toString(text, cb) and toString(text, opts, cb) / (text, opts). - if (typeof opts === "function") { - cb = opts; - opts = undefined; - } - var merged = {}; - if (opts && typeof opts === "object") { - for (var key in opts) { - if (Object.prototype.hasOwnProperty.call(opts, key)) merged[key] = opts[key]; - } - } - // Only the terminal renderer has the oversize problem. `type` defaults - // to "utf8" in the qrcode package, but the WhatsApp path always passes - // "terminal" explicitly; force small there and leave every other type - // (svg/png/utf8 data URIs used elsewhere) exactly as the caller asked. - if (merged.type === "terminal") { - merged.small = true; - } - return origToString.call(this, text, merged, cb); - }; - markPatched(mod); - return mod; +// Install the Module._load hook that patches qrcode / qrcode-terminal on load. +// Guarded so double-require is a no-op. Runs on import (the file is loaded via +// `--require`/preload), preserving the previous self-installing IIFE behavior. +function installWhatsappQrCompactHook() { + if (process.__nemoclawWhatsappQrCompactInstalled) return; + try { + Object.defineProperty(process, "__nemoclawWhatsappQrCompactInstalled", { value: true }); + } catch (_e) { + process.__nemoclawWhatsappQrCompactInstalled = true; } - function patchQrcodeTerminal(mod) { - if (mod.__nemoclawCompactPatched) return mod; - var origGenerate = mod.generate; - mod.generate = function (text, opts, cb) { - if (typeof opts === "function") { - cb = opts; - opts = undefined; - } - var merged = {}; - if (opts && typeof opts === "object") { - for (var key in opts) { - if (Object.prototype.hasOwnProperty.call(opts, key)) merged[key] = opts[key]; - } - } - merged.small = true; - return origGenerate.call(this, text, merged, cb); - }; - markPatched(mod); - return mod; - } + var Module = require("module"); + var origLoad = Module._load; Module._load = function (request, _parent, _isMain) { var loaded = origLoad.apply(this, arguments); - // Cheap path filter: only inspect modules whose request mentions qrcode. - // `import("qrcode")` arrives here as the resolved absolute path - // (…/qrcode/lib/index.js), so match on the path segment too, not just the - // bare specifier. - if (typeof request === "string" && request.indexOf("qrcode") !== -1) { - try { - if (isQrcodePackage(loaded)) return patchQrcode(loaded); - if (isQrcodeTerminalPackage(loaded)) return patchQrcodeTerminal(loaded); - } catch (_e) { - return loaded; - } - } - return loaded; + // Cheap path filter + shape-detect routing. `import("qrcode")` arrives here + // as the resolved absolute path (…/qrcode/lib/index.js), so the filter in + // resolvePatchedModule matches on the path segment too, not just the bare + // specifier. + return resolvePatchedModule(request, loaded); }; -})(); +} + +installWhatsappQrCompactHook(); diff --git a/src/lib/onboard/dashboard-access.test.ts b/src/lib/onboard/dashboard-access.test.ts index 776b8cb94b2..9b41e7cb621 100644 --- a/src/lib/onboard/dashboard-access.test.ts +++ b/src/lib/onboard/dashboard-access.test.ts @@ -1,10 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { buildAuthenticatedDashboardUrl, + buildDashboardChain, dashboardUrlForDisplay, getDashboardAccessInfo, getDashboardForwardPort, @@ -77,3 +78,79 @@ describe("dashboard access helpers", () => { ]); }); }); + +// The pure buildChain({ bindOverride }) decision is covered in +// src/lib/dashboard/contract.test.ts. These tests pin the I/O boundary: +// readBindOverride() reads NEMOCLAW_DASHBOARD_BIND from the env and +// buildDashboardChain wires it into buildChain. The dangerous NEGATIVE cases +// (invalid / loopback values must NOT open a remote bind) were previously only +// asserted in the live dashboard-remote-bind E2E. +describe("NEMOCLAW_DASHBOARD_BIND remote-bind opt-in gate (#3259)", () => { + const LOOPBACK_URL = "http://127.0.0.1:18789"; + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("opens the remote bind when env NEMOCLAW_DASHBOARD_BIND=0.0.0.0", () => { + const chain = buildDashboardChain(LOOPBACK_URL, { + env: { NEMOCLAW_DASHBOARD_BIND: "0.0.0.0" }, + }); + expect(chain.bindAddress).toBe("0.0.0.0"); + expect(chain.forwardTarget).toBe("0.0.0.0:18789"); + expect( + getDashboardForwardTarget(LOOPBACK_URL, { env: { NEMOCLAW_DASHBOARD_BIND: "0.0.0.0" } }), + ).toBe("0.0.0.0:18789"); + }); + + it("stays loopback when env NEMOCLAW_DASHBOARD_BIND is unset", () => { + const chain = buildDashboardChain(LOOPBACK_URL, { env: {} }); + expect(chain.bindAddress).toBe("127.0.0.1"); + expect(chain.forwardTarget).toBe("18789"); + }); + + it("stays loopback when env NEMOCLAW_DASHBOARD_BIND is empty", () => { + const chain = buildDashboardChain(LOOPBACK_URL, { + env: { NEMOCLAW_DASHBOARD_BIND: "" }, + }); + expect(chain.bindAddress).toBe("127.0.0.1"); + expect(chain.forwardTarget).toBe("18789"); + }); + + it("stays loopback when env NEMOCLAW_DASHBOARD_BIND=127.0.0.1", () => { + const chain = buildDashboardChain(LOOPBACK_URL, { + env: { NEMOCLAW_DASHBOARD_BIND: "127.0.0.1" }, + }); + expect(chain.bindAddress).toBe("127.0.0.1"); + expect(chain.forwardTarget).toBe("18789"); + }); + + it.each([ + "0.0.0.0; rm -rf", + "1.2.3.4", + "true", + "10.0.0.5", + " 0.0.0.0", + "0.0.0.0 ", + ])("does NOT open a remote bind for invalid env value %j", (value) => { + const chain = buildDashboardChain(LOOPBACK_URL, { + env: { NEMOCLAW_DASHBOARD_BIND: value }, + }); + expect(chain.bindAddress).toBe("127.0.0.1"); + expect(chain.forwardTarget).toBe("18789"); + }); + + it("falls back to process.env when no options.env override is provided", () => { + vi.stubEnv("NEMOCLAW_DASHBOARD_BIND", "0.0.0.0"); + const chain = buildDashboardChain(LOOPBACK_URL); + expect(chain.bindAddress).toBe("0.0.0.0"); + expect(chain.forwardTarget).toBe("0.0.0.0:18789"); + }); + + it("does NOT open a remote bind for invalid process.env value", () => { + vi.stubEnv("NEMOCLAW_DASHBOARD_BIND", "0.0.0.0; rm -rf"); + const chain = buildDashboardChain(LOOPBACK_URL); + expect(chain.bindAddress).toBe("127.0.0.1"); + expect(chain.forwardTarget).toBe("18789"); + }); +}); diff --git a/src/lib/sandbox/config-get.test.ts b/src/lib/sandbox/config-get.test.ts new file mode 100644 index 00000000000..6e0b39c65e4 --- /dev/null +++ b/src/lib/sandbox/config-get.test.ts @@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Output-assembly contract for `nemoclaw config get [--key ...]`. +// +// This pins the two invariants the command owes the operator, both of which +// live in configGet's own assembly step rather than in the shared credential +// filter (whose field detection is covered by credential-filter.test.ts): +// +// 1. No credential-shaped value ever reaches stdout — provider keys +// (`nvapi-`, `sk-`), `Bearer ` tokens, etc. are stripped by +// stripCredentials before printing (whole config AND a nested --key view). +// 2. The `gateway` field is dropped entirely, because it holds runtime +// auth material regenerated at gateway launch. +// +// The class of gap: an `nvapi-` credential-format assertion that previously +// only existed in a live E2E test, so a regression here shipped unnoticed. We +// drive the real configGet through a stubbed openshell read + captured stdout. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +// The shared source-require hook compiles the TypeScript sources into the same +// writable CommonJS cache these modules already share, so replacing the +// openshell client's capture export before requiring ./config makes configGet's +// internal read return our fixture instead of shelling out to a real sandbox. +const clientModulePath = require.resolve("../adapters/openshell/client"); +const configModulePath = require.resolve("./config"); + +type CaptureResult = { + status: number; + signal: null; + error?: undefined; + stdout: string; + output: string; + stderr: string; +}; + +const client = require(clientModulePath) as { + captureOpenshellCommand: (...args: unknown[]) => CaptureResult; +}; +const realCapture = client.captureOpenshellCommand; + +// The raw config the fake sandbox `cat` returns. It carries every secret +// shape the redaction contract must strip plus a gateway block that must be +// omitted wholesale, alongside benign fields that must survive untouched. +const SANDBOX_CONFIG = { + model: { id: "nvidia/nemotron-3", temperature: 0.2 }, + provider: { + // Low-entropy, obviously-fake fixtures (sequential alphabet) so the secret + // scanner does not flag them while they still match the redaction patterns. + apiKey: "nvapi-abcdefghijklmnopqrstuvwxyz0123456789", + baseUrl: "https://inference.nvidia.com/v1", + }, + openaiCompat: { apiKey: "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789" }, + mcp: { + remote: { headers: { authorization: "Bearer super-secret-token-value" } }, + }, + gateway: { + token: "nvapi-gateway000000000000000000000000000000", + url: "http://127.0.0.1:8080", + }, +}; + +function loadConfigGet(): (name: string, opts?: { key?: string; format?: string }) => void { + delete require.cache[configModulePath]; + const mod = require(configModulePath) as { + configGet: (name: string, opts?: { key?: string; format?: string }) => void; + }; + return mod.configGet; +} + +function stubSandboxRead(rawConfig: unknown): void { + const raw = JSON.stringify(rawConfig); + client.captureOpenshellCommand = () => ({ + status: 0, + signal: null, + stdout: raw, + output: raw, + stderr: "", + }); +} + +function captureStdout(run: () => void): string { + const chunks: string[] = []; + const spy = vi.spyOn(console, "log").mockImplementation((...args: unknown[]) => { + chunks.push(args.map((a) => (typeof a === "string" ? a : String(a))).join(" ")); + }); + try { + run(); + } finally { + spy.mockRestore(); + } + return chunks.join("\n"); +} + +describe("configGet output redaction and gateway omission (#config-get)", () => { + beforeEach(() => { + stubSandboxRead(SANDBOX_CONFIG); + }); + + afterEach(() => { + client.captureOpenshellCommand = realCapture; + delete require.cache[configModulePath]; + }); + + it("never prints nvapi-, sk-, or Bearer credential values in the full config", () => { + const configGet = loadConfigGet(); + const out = captureStdout(() => configGet("alpha")); + + expect(out).not.toMatch(/nvapi-/); + expect(out).not.toMatch(/sk-proj-/); + expect(out).not.toMatch(/Bearer super-secret-token-value/); + expect(out).not.toContain("super-secret-token-value"); + }); + + it("omits the gateway field entirely from the full config", () => { + const configGet = loadConfigGet(); + const out = captureStdout(() => configGet("alpha")); + + const parsed = JSON.parse(out) as Record; + expect(parsed).not.toHaveProperty("gateway"); + }); + + it("passes non-secret fields through unredacted", () => { + const configGet = loadConfigGet(); + const out = captureStdout(() => configGet("alpha")); + + const parsed = JSON.parse(out) as { + model: { id: string; temperature: number }; + provider: { baseUrl: string }; + }; + expect(parsed.model.id).toBe("nvidia/nemotron-3"); + expect(parsed.model.temperature).toBe(0.2); + // The provider URL is not a credential and must survive redaction. + expect(parsed.provider.baseUrl).toBe("https://inference.nvidia.com/v1"); + }); + + it("redacts a credential reached through a nested --key path", () => { + const configGet = loadConfigGet(); + const out = captureStdout(() => configGet("alpha", { key: "provider.apiKey" })); + + expect(out).not.toMatch(/nvapi-/); + expect(out).toContain("[STRIPPED_BY_MIGRATION]"); + }); + + it("returns the leaf value for a non-secret --key path", () => { + const configGet = loadConfigGet(); + const out = captureStdout(() => configGet("alpha", { key: "model.id" })); + + expect(JSON.parse(out)).toBe("nvidia/nemotron-3"); + }); + + it("refuses to expose the gateway section via --key gateway (#config-get)", () => { + const configGet = loadConfigGet(); + // gateway is deleted before dotpath extraction, so the key is not found and + // the command fails rather than leaking regenerated auth material. + expect(() => configGet("alpha", { key: "gateway.token" })).toThrow(/not found/i); + }); +}); diff --git a/src/lib/shields/audit-format.test.ts b/src/lib/shields/audit-format.test.ts index b0f3864a88c..a25b5f133fa 100644 --- a/src/lib/shields/audit-format.test.ts +++ b/src/lib/shields/audit-format.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; // Test the audit entry format and JSONL structure using the same logic // as the production module but with a controllable output path. @@ -116,3 +116,90 @@ describe("shields-audit format", () => { expect(line).not.toContain("sk-"); }); }); + +// Pin the PRODUCTION appendAuditEntry (not an inline reimplementation): the +// real writer must strip credential values from every serialized record kind. +// This closes the gap where only the live shields-config E2E asserted that the +// on-disk shields-audit.jsonl never persists secrets. The real module captures +// its AUDIT_FILE path from resolveNemoclawStateDir(process.env.HOME) at load +// time, so each case points HOME at a temp dir and re-imports for a fresh path. +describe("shields-audit production redaction", () => { + let homeDir: string; + let realAuditPath: string; + let savedHome: string | undefined; + + beforeEach(() => { + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-audit-home-")); + realAuditPath = path.join(homeDir, ".nemoclaw", "state", "shields-audit.jsonl"); + savedHome = process.env.HOME; + process.env.HOME = homeDir; + vi.resetModules(); + }); + + afterEach(() => { + delete process.env.HOME; + Object.assign(process.env, savedHome === undefined ? {} : { HOME: savedHome }); + vi.resetModules(); + fs.rmSync(homeDir, { recursive: true, force: true }); + }); + + async function loadAppendAuditEntry() { + const mod = await import("./audit"); + return mod.appendAuditEntry; + } + + const SECRETS = { + nvapi: "nvapi-abcdefghijklmnopqrstuvwxyz0123456789", + sk: "sk-abcdefghijklmnopqrstuvwxyz0123456789", + bearer: "Bearer abcdefghijklmnopqrstuvwxyz0123456789", + } as const; + + function assertNoSecrets(line: string) { + expect(line).not.toContain(SECRETS.nvapi); + expect(line).not.toContain(SECRETS.sk); + expect(line).not.toContain(SECRETS.bearer); + expect(line).not.toContain("nvapi-a"); + expect(line).not.toContain("sk-abcdef"); + } + + it.each([ + "shields_down", + "shields_up", + "shields_auto_restore", + ] as const)("strips nvapi-/sk-/Bearer secrets from the free-text reason of %s records", async (action) => { + const appendAuditEntry = await loadAppendAuditEntry(); + appendAuditEntry({ + action, + sandbox: "openclaw", + timestamp: "2026-04-13T14:30:00Z", + reason: `key=${SECRETS.nvapi} also ${SECRETS.sk} and ${SECRETS.bearer}`, + }); + + const line = fs.readFileSync(realAuditPath, "utf-8").trim(); + assertNoSecrets(line); + // The line must still be a valid, parseable JSONL entry after redaction. + const entry = JSON.parse(line); + expect(entry.action).toBe(action); + expect(entry.sandbox).toBe("openclaw"); + }); + + it("strips secrets from the error field while preserving benign fields", async () => { + const appendAuditEntry = await loadAppendAuditEntry(); + appendAuditEntry({ + action: "shields_up_failed", + sandbox: "hermes", + timestamp: "2026-04-13T14:30:00Z", + error: `guard failed using ${SECRETS.nvapi} / ${SECRETS.bearer}`, + reason: `retry with ${SECRETS.sk}`, + policy_applied: "permissive", + }); + + const line = fs.readFileSync(realAuditPath, "utf-8").trim(); + assertNoSecrets(line); + const entry = JSON.parse(line); + // Structured, non-secret fields survive redaction verbatim. + expect(entry.sandbox).toBe("hermes"); + expect(entry.policy_applied).toBe("permissive"); + expect(entry.action).toBe("shields_up_failed"); + }); +}); diff --git a/test/credential-rotation.test.ts b/test/credential-rotation.test.ts index 3afb948f1b9..665de3c12e7 100644 --- a/test/credential-rotation.test.ts +++ b/test/credential-rotation.test.ts @@ -246,4 +246,136 @@ describe("credential rotation detection", () => { vi.restoreAllMocks(); }); }); + + // The selective-rebuild contract: when only a subset of messaging credentials + // rotate, the provider-name list that drives the user-facing + // "Messaging credential(s) rotated: …" line and the rebuild set must name + // ONLY the changed provider(s) — never their unchanged siblings. onboard.ts + // renders this via `credentialRotation.changedProviders.join(", ")`, so these + // cases assert on that exact provider-name selection rather than the boolean + // rotation / hash logic covered above. + describe("selective-rebuild provider naming", () => { + // Three sibling providers sharing a single stored plan; each case rotates a + // different subset and asserts the resulting name list. + function threeProviderPlan(hashes: { telegram: string; discord: string; slack: string }) { + return makePlanEntry("multi-sandbox", [ + { providerEnvKey: "TELEGRAM_BOT_TOKEN", credentialHash: hashes.telegram }, + { providerEnvKey: "DISCORD_BOT_TOKEN", credentialHash: hashes.discord }, + { providerEnvKey: "SLACK_BOT_TOKEN", credentialHash: hashes.slack }, + ]); + } + + const A = "multi-telegram-bridge"; + const B = "multi-discord-bridge"; + const C = "multi-slack-bridge"; + + it("names ONLY provider A and excludes unchanged siblings B and C", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue( + threeProviderPlan({ + telegram: hashCredentialOrThrow("tg-old"), + discord: hashCredentialOrThrow("dc-same"), + slack: hashCredentialOrThrow("sl-same"), + }), + ); + + const result = detectMessagingCredentialRotation("multi-sandbox", [ + { name: A, envKey: "TELEGRAM_BOT_TOKEN", token: "tg-new" }, + { name: B, envKey: "DISCORD_BOT_TOKEN", token: "dc-same" }, + { name: C, envKey: "SLACK_BOT_TOKEN", token: "sl-same" }, + ]); + + expect(result.changed).toBe(true); + // Rebuild set / message name only the rotated provider. + expect(result.changedProviders).toEqual([A]); + expect(result.changedProviders).not.toContain(B); + expect(result.changedProviders).not.toContain(C); + // The exact user-facing string driven by this list. + expect(result.changedProviders.join(", ")).toBe(A); + vi.restoreAllMocks(); + }); + + it("names a middle sibling only, leaving A and C out of the rebuild set", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue( + threeProviderPlan({ + telegram: hashCredentialOrThrow("tg-same"), + discord: hashCredentialOrThrow("dc-old"), + slack: hashCredentialOrThrow("sl-same"), + }), + ); + + const result = detectMessagingCredentialRotation("multi-sandbox", [ + { name: A, envKey: "TELEGRAM_BOT_TOKEN", token: "tg-same" }, + { name: B, envKey: "DISCORD_BOT_TOKEN", token: "dc-new" }, + { name: C, envKey: "SLACK_BOT_TOKEN", token: "sl-same" }, + ]); + + expect(result.changedProviders).toEqual([B]); + expect(result.changedProviders.join(", ")).toBe(B); + vi.restoreAllMocks(); + }); + + it("names all changed providers when multiple siblings rotate, preserving order", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue( + threeProviderPlan({ + telegram: hashCredentialOrThrow("tg-old"), + discord: hashCredentialOrThrow("dc-same"), + slack: hashCredentialOrThrow("sl-old"), + }), + ); + + const result = detectMessagingCredentialRotation("multi-sandbox", [ + { name: A, envKey: "TELEGRAM_BOT_TOKEN", token: "tg-new" }, + { name: B, envKey: "DISCORD_BOT_TOKEN", token: "dc-same" }, + { name: C, envKey: "SLACK_BOT_TOKEN", token: "sl-new" }, + ]); + + expect(result.changed).toBe(true); + // Both changed siblings named, in tokenDefs order; unchanged B omitted. + expect(result.changedProviders).toEqual([A, C]); + expect(result.changedProviders).not.toContain(B); + expect(result.changedProviders.join(", ")).toBe(`${A}, ${C}`); + vi.restoreAllMocks(); + }); + + it("names every provider when all siblings rotate", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue( + threeProviderPlan({ + telegram: hashCredentialOrThrow("tg-old"), + discord: hashCredentialOrThrow("dc-old"), + slack: hashCredentialOrThrow("sl-old"), + }), + ); + + const result = detectMessagingCredentialRotation("multi-sandbox", [ + { name: A, envKey: "TELEGRAM_BOT_TOKEN", token: "tg-new" }, + { name: B, envKey: "DISCORD_BOT_TOKEN", token: "dc-new" }, + { name: C, envKey: "SLACK_BOT_TOKEN", token: "sl-new" }, + ]); + + expect(result.changedProviders).toEqual([A, B, C]); + expect(result.changedProviders.join(", ")).toBe(`${A}, ${B}, ${C}`); + vi.restoreAllMocks(); + }); + + it("produces an empty name list when no sibling rotates (no rebuild, no message)", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue( + threeProviderPlan({ + telegram: hashCredentialOrThrow("tg-same"), + discord: hashCredentialOrThrow("dc-same"), + slack: hashCredentialOrThrow("sl-same"), + }), + ); + + const result = detectMessagingCredentialRotation("multi-sandbox", [ + { name: A, envKey: "TELEGRAM_BOT_TOKEN", token: "tg-same" }, + { name: B, envKey: "DISCORD_BOT_TOKEN", token: "dc-same" }, + { name: C, envKey: "SLACK_BOT_TOKEN", token: "sl-same" }, + ]); + + expect(result.changed).toBe(false); + expect(result.changedProviders).toEqual([]); + expect(result.changedProviders.join(", ")).toBe(""); + vi.restoreAllMocks(); + }); + }); }); diff --git a/test/e2e/live/common-egress-agent-helpers.ts b/test/e2e/live/common-egress-agent-helpers.ts new file mode 100644 index 00000000000..fc7e783bd5e --- /dev/null +++ b/test/e2e/live/common-egress-agent-helpers.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Pure parsing/classification helpers shared by the common-egress-agent live +// E2E target and its PR-collected unit tests. Extracting them lets the fast +// e2e-support project verify the OpenClaw JSON framing, Hermes response parsing, +// expected-token matching, and pre-contract provider-validation skip +// classification without gating on NEMOCLAW_RUN_LIVE_E2E=1. + +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; + +interface AgentJsonDoc { + payloads?: Array<{ text?: unknown }>; + result?: { payloads?: Array<{ text?: unknown }> }; +} + +interface ChatCompletionLike { + choices?: Array<{ + message?: { + content?: unknown; + reasoning_content?: unknown; + }; + text?: unknown; + }>; +} + +export interface CommonEgressProviderValidationSkip { + http429ProviderValidationFailure: boolean; + matches: boolean; + sanitizedEndpointValidationFailure: boolean; + transientProviderValidationFailure: boolean; +} + +export function text(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function parseAgentJsonDocs(raw: string): AgentJsonDoc[] { + try { + const parsed = JSON.parse(raw) as AgentJsonDoc | AgentJsonDoc[]; + return Array.isArray(parsed) ? parsed : [parsed]; + } catch { + // Invalid state: `openclaw agent --json` has emitted both single JSON + // documents and log-prefixed streams across versions. Source boundary: + // OpenClaw CLI stdout framing inside the sandbox, outside this NemoClaw + // migration. Source-fix constraint: keep this test local and legacy-script + // compatible instead of rewriting shared fixtures or patching OpenClaw from + // a migration PR. Removal condition: supported OpenClaw versions guarantee + // a strict single JSON document with payload text on stdout. + } + + const docs: AgentJsonDoc[] = []; + for (let index = 0; index < raw.length; index += 1) { + if (raw[index] !== "{") continue; + for (let end = index + 1; end <= raw.length; end += 1) { + try { + const parsed = JSON.parse(raw.slice(index, end)) as AgentJsonDoc | AgentJsonDoc[]; + docs.push(...(Array.isArray(parsed) ? parsed : [parsed])); + index = end - 1; + break; + } catch { + // Keep extending the candidate slice until it becomes valid JSON. + } + } + } + return docs; +} + +export function parseOpenClawAgentText(raw: string): string { + return parseAgentJsonDocs(raw) + .flatMap((doc) => doc.payloads ?? doc.result?.payloads ?? []) + .map((payload) => payload.text) + .filter((value): value is string => typeof value === "string") + .join("\n") + .trim(); +} + +export function parseChatContent(raw: string): string { + const doc = JSON.parse(raw) as ChatCompletionLike; + const choice = doc.choices?.[0]; + const content = choice?.message?.content ?? choice?.message?.reasoning_content ?? choice?.text; + return typeof content === "string" ? content.trim() : ""; +} + +function compactAgentReply(value: string): string { + return value.replace(/\s+/gu, ""); +} + +export function agentReplyContainsToken(reply: string, expected: string): boolean { + const compactExpected = compactAgentReply(expected); + return compactExpected.length > 0 && compactAgentReply(reply).includes(compactExpected); +} + +export function classifyPreContractProviderValidationSkip( + result: Pick, +): CommonEgressProviderValidationSkip { + const output = text(result); + const providerValidation = + /endpoint validation failed|failed to verify inference endpoint|Chat Completions API validation/i.test( + output, + ); + const transientProviderValidationFailure = isTransientProviderValidationFailure(result); + const http429ProviderValidationFailure = + providerValidation && /HTTP\s*429|\b429\b|rate[- ]?limit|too many requests/i.test(output); + const sanitizedEndpointValidationFailure = + providerValidation && + /Validation details were omitted to avoid exposing credentials/i.test(output) && + process.env.GITHUB_ACTIONS === "true"; + + return { + http429ProviderValidationFailure, + matches: + transientProviderValidationFailure || + http429ProviderValidationFailure || + sanitizedEndpointValidationFailure, + sanitizedEndpointValidationFailure, + transientProviderValidationFailure, + }; +} diff --git a/test/e2e/live/common-egress-agent.test.ts b/test/e2e/live/common-egress-agent.test.ts index bdec78a04a8..5e3da826d74 100644 --- a/test/e2e/live/common-egress-agent.test.ts +++ b/test/e2e/live/common-egress-agent.test.ts @@ -25,8 +25,13 @@ import { import { shouldRunLiveE2E } from "../fixtures/live-project-gate.ts"; import type { SecretStore } from "../fixtures/secrets.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + agentReplyContainsToken, + classifyPreContractProviderValidationSkip, + parseChatContent, + parseOpenClawAgentText, +} from "./common-egress-agent-helpers.ts"; import { stripAnsi } from "./json-envelope.ts"; -import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; // // Preserve the legacy live boundary: real NemoClaw onboard, real OpenShell @@ -62,28 +67,6 @@ validateSandboxName(HERMES_SANDBOX); type NemoEnv = NodeJS.ProcessEnv; type SkipFn = (note?: string) => never; -interface AgentJsonDoc { - payloads?: Array<{ text?: unknown }>; - result?: { payloads?: Array<{ text?: unknown }> }; -} - -interface ChatCompletionLike { - choices?: Array<{ - message?: { - content?: unknown; - reasoning_content?: unknown; - }; - text?: unknown; - }>; -} - -interface CommonEgressProviderValidationSkip { - http429ProviderValidationFailure: boolean; - matches: boolean; - sanitizedEndpointValidationFailure: boolean; - transientProviderValidationFailure: boolean; -} - interface CleanupAttempt { exitCode: number | null; missingSandboxTolerated: boolean; @@ -114,62 +97,6 @@ function commandEnv(extra: NemoEnv = {}): NemoEnv { }; } -function parseAgentJsonDocs(raw: string): AgentJsonDoc[] { - try { - const parsed = JSON.parse(raw) as AgentJsonDoc | AgentJsonDoc[]; - return Array.isArray(parsed) ? parsed : [parsed]; - } catch { - // Invalid state: `openclaw agent --json` has emitted both single JSON - // documents and log-prefixed streams across versions. Source boundary: - // OpenClaw CLI stdout framing inside the sandbox, outside this NemoClaw - // migration. Source-fix constraint: keep this test local and legacy-script - // compatible instead of rewriting shared fixtures or patching OpenClaw from - // a migration PR. Removal condition: supported OpenClaw versions guarantee - // a strict single JSON document with payload text on stdout. - } - - const docs: AgentJsonDoc[] = []; - for (let index = 0; index < raw.length; index += 1) { - if (raw[index] !== "{") continue; - for (let end = index + 1; end <= raw.length; end += 1) { - try { - const parsed = JSON.parse(raw.slice(index, end)) as AgentJsonDoc | AgentJsonDoc[]; - docs.push(...(Array.isArray(parsed) ? parsed : [parsed])); - index = end - 1; - break; - } catch { - // Keep extending the candidate slice until it becomes valid JSON. - } - } - } - return docs; -} - -function parseOpenClawAgentText(raw: string): string { - return parseAgentJsonDocs(raw) - .flatMap((doc) => doc.payloads ?? doc.result?.payloads ?? []) - .map((payload) => payload.text) - .filter((value): value is string => typeof value === "string") - .join("\n") - .trim(); -} - -function parseChatContent(raw: string): string { - const doc = JSON.parse(raw) as ChatCompletionLike; - const choice = doc.choices?.[0]; - const content = choice?.message?.content ?? choice?.message?.reasoning_content ?? choice?.text; - return typeof content === "string" ? content.trim() : ""; -} - -function compactAgentReply(value: string): string { - return value.replace(/\s+/gu, ""); -} - -function agentReplyContainsToken(reply: string, expected: string): boolean { - const compactExpected = compactAgentReply(expected); - return compactExpected.length > 0 && compactAgentReply(reply).includes(compactExpected); -} - function httpStatusFromResponse(raw: string): string { return ( raw @@ -205,33 +132,6 @@ function isOpenClawTransientAgentError(output: string): boolean { ); } -function classifyPreContractProviderValidationSkip( - result: Pick, -): CommonEgressProviderValidationSkip { - const output = text(result); - const providerValidation = - /endpoint validation failed|failed to verify inference endpoint|Chat Completions API validation/i.test( - output, - ); - const transientProviderValidationFailure = isTransientProviderValidationFailure(result); - const http429ProviderValidationFailure = - providerValidation && /HTTP\s*429|\b429\b|rate[- ]?limit|too many requests/i.test(output); - const sanitizedEndpointValidationFailure = - providerValidation && - /Validation details were omitted to avoid exposing credentials/i.test(output) && - process.env.GITHUB_ACTIONS === "true"; - - return { - http429ProviderValidationFailure, - matches: - transientProviderValidationFailure || - http429ProviderValidationFailure || - sanitizedEndpointValidationFailure, - sanitizedEndpointValidationFailure, - transientProviderValidationFailure, - }; -} - function isMissingSandboxOutput(output: string): boolean { return /Sandbox .* does not exist|sandbox .* does not exist|does not exist|not found|No such sandbox/i.test( output, @@ -666,83 +566,6 @@ const openClawTest = process.env.NEMOCLAW_COMMON_EGRESS_SKIP_OPENCLAW === "1" ? test.skip : liveTest; const hermesTest = process.env.NEMOCLAW_COMMON_EGRESS_SKIP_HERMES === "1" ? test.skip : liveTest; -test("common-egress agent OpenClaw JSON parser accepts framed agent payloads", () => { - expect( - parseOpenClawAgentText( - JSON.stringify({ payloads: [{ text: "noise" }, { text: "WEATHER_AGENT_OK" }] }), - ), - ).toContain("WEATHER_AGENT_OK"); - expect( - parseOpenClawAgentText( - JSON.stringify({ result: { payloads: [{ text: "REFERENCE_AGENT_OK" }] } }), - ), - ).toContain("REFERENCE_AGENT_OK"); - expect( - parseOpenClawAgentText( - `openclaw log line\n${JSON.stringify({ - result: { payloads: [{ text: "HERMES_REFERENCE_AGENT_OK" }] }, - })}\n`, - ), - ).toContain("HERMES_REFERENCE_AGENT_OK"); -}); - -test("common-egress agent Hermes response parser reads message content", () => { - expect( - parseChatContent( - JSON.stringify({ choices: [{ message: { content: "HERMES_REFERENCE_AGENT_OK" } }] }), - ), - ).toBe("HERMES_REFERENCE_AGENT_OK"); -}); - -test("common-egress agent expected-token matching ignores model line breaks", () => { - expect(agentReplyContainsToken("REFER\nENCE_AGENT_OK", "REFERENCE_AGENT_OK")).toBe(true); - expect(agentReplyContainsToken("HERMES_REFERENCE\n_AGENT_OK", "HERMES_REFERENCE_AGENT_OK")).toBe( - true, - ); -}); - -test("common-egress agent classifies pre-contract provider validation skips", () => { - expect( - classifyPreContractProviderValidationSkip({ - stdout: "", - stderr: - "NVIDIA Endpoints endpoint validation failed.\nChat Completions API validation returned HTTP 429", - }), - ).toMatchObject({ - http429ProviderValidationFailure: true, - matches: true, - }); - - const originalGithubActions = process.env.GITHUB_ACTIONS; - try { - process.env.GITHUB_ACTIONS = "true"; - expect( - classifyPreContractProviderValidationSkip({ - stdout: "", - stderr: - "NVIDIA Endpoints endpoint validation failed.\nValidation details were omitted to avoid exposing credentials.", - }), - ).toMatchObject({ - matches: true, - sanitizedEndpointValidationFailure: true, - }); - } finally { - if (originalGithubActions === undefined) { - delete process.env.GITHUB_ACTIONS; - } else { - process.env.GITHUB_ACTIONS = originalGithubActions; - } - } - - expect( - classifyPreContractProviderValidationSkip({ - stdout: "", - stderr: - "NVIDIA Endpoints endpoint validation failed.\ninvalid NVIDIA_INFERENCE_API_KEY credential", - }), - ).toMatchObject({ matches: false }); -}); - describe.sequential("common-egress agent live targets", () => { openClawTest( "C1 OpenClaw balanced excludes weather until explicitly added, then permits a verified wttr.in curl", diff --git a/test/e2e/live/messaging-compatible-endpoint.test.ts b/test/e2e/live/messaging-compatible-endpoint.test.ts index 4d6002dbc6d..f81dfc1b664 100644 --- a/test/e2e/live/messaging-compatible-endpoint.test.ts +++ b/test/e2e/live/messaging-compatible-endpoint.test.ts @@ -15,13 +15,15 @@ import http from "node:http"; import type { AddressInfo } from "node:net"; import path from "node:path"; -import { describe, it } from "vitest"; - import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2E } from "../fixtures/live-project-gate.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + COMPAT_AGENT_PROMPT, + COMPAT_AGENT_REPLY, +} from "../support/messaging-endpoint-classifiers.ts"; import { cleanupMessagingState, commandEnv, @@ -55,10 +57,6 @@ const HOP_BY_HOP_HEADERS = new Set([ "transfer-encoding", "upgrade", ]); -const COMPAT_AGENT_REPLY = "COMPAT_MOCK_ROUTE_5098_OK"; -const COMPAT_AGENT_PROMPT = - "Call the configured model and report the compatible endpoint route token."; - function nodeEvalArg(source: string): string { const encoded = Buffer.from(source, "utf8").toString("base64"); return `eval(Buffer.from(${JSON.stringify(encoded)}, "base64").toString("utf8"))`; @@ -608,18 +606,6 @@ async function assertOpenClawAgentTurn( expect(leaked, `Proxy hop headers leaked to upstream: ${leaked.join(",")}`).toEqual([]); } -describe("messaging-compatible-endpoint live test local classifiers", () => { - it("does not satisfy the agent reply assertion with echoed prompt text", () => { - expect(COMPAT_AGENT_PROMPT).not.toContain(COMPAT_AGENT_REPLY); - expect( - parseOpenClawAgentText(JSON.stringify({ result: { content: COMPAT_AGENT_PROMPT } })), - ).not.toContain(COMPAT_AGENT_REPLY); - expect( - parseOpenClawAgentText(JSON.stringify({ result: { content: COMPAT_AGENT_REPLY } })), - ).toContain(COMPAT_AGENT_REPLY); - }); -}); - liveTest( "messaging compatible endpoint routes Telegram-enabled OpenClaw through inference.local", { timeout: TEST_TIMEOUT_MS }, diff --git a/test/e2e/live/openclaw-inference-switch-helpers.ts b/test/e2e/live/openclaw-inference-switch-helpers.ts new file mode 100644 index 00000000000..7d664856e62 --- /dev/null +++ b/test/e2e/live/openclaw-inference-switch-helpers.ts @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Pure reply-matching helper shared by the openclaw-inference-switch live E2E +// target and its PR-collected unit test. Extracting the predicate lets the fast +// e2e-support project verify that a wrapped/whitespace-split "PONG" reply is +// accepted while echoed or embedded tokens are rejected, without gating on +// NEMOCLAW_RUN_LIVE_E2E=1. + +export function agentReplyContainsToken(reply: string, expected: string): boolean { + const normalizedReply = reply.replace(/\s+/gu, "").toUpperCase(); + const normalizedExpected = expected.replace(/\s+/gu, "").toUpperCase(); + return normalizedExpected.length > 0 && normalizedReply === normalizedExpected; +} + +// Baseline (mock-Anthropic) inference config the live target builds when +// NEMOCLAW_SWITCH_MOCK_ANTHROPIC=1 points OpenClaw at a local fake OpenAI- +// compatible server. Extracted so the fast e2e-support project can assert the +// exact env wiring (credential, model, endpoint, preferred API, provider) +// without gating on NEMOCLAW_RUN_LIVE_E2E=1. +export const MOCK_BASELINE_API_KEY = "openclaw-switch-baseline-credential"; +export const MOCK_BASELINE_MODEL = "openclaw-switch-baseline-model"; + +export interface BaselineInferenceConfig { + apiKey: string; + endpointUrl: string; + env: NodeJS.ProcessEnv; +} + +export function mockBaselineInference(endpointUrl: string): BaselineInferenceConfig { + return { + apiKey: MOCK_BASELINE_API_KEY, + endpointUrl, + env: { + COMPATIBLE_API_KEY: MOCK_BASELINE_API_KEY, + NEMOCLAW_COMPAT_MODEL: MOCK_BASELINE_MODEL, + NEMOCLAW_ENDPOINT_URL: endpointUrl, + NEMOCLAW_MODEL: MOCK_BASELINE_MODEL, + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + }, + }; +} diff --git a/test/e2e/live/openclaw-inference-switch.test.ts b/test/e2e/live/openclaw-inference-switch.test.ts index 32a2e8ad1bd..5321e34ccb9 100644 --- a/test/e2e/live/openclaw-inference-switch.test.ts +++ b/test/e2e/live/openclaw-inference-switch.test.ts @@ -37,6 +37,12 @@ import { } from "../fixtures/inference-switch-retry.ts"; import { shouldRunLiveE2E } from "../fixtures/live-project-gate.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + agentReplyContainsToken, + MOCK_BASELINE_API_KEY, + MOCK_BASELINE_MODEL, + mockBaselineInference, +} from "./openclaw-inference-switch-helpers.ts"; import { PUBLIC_NVIDIA_SWITCH_MODEL, PUBLIC_NVIDIA_SWITCH_PROVIDER, @@ -53,8 +59,6 @@ const SWITCH_MODEL = process.env.NEMOCLAW_SWITCH_MODEL ?? PUBLIC_NVIDIA_SWITCH_M const SWITCH_INFERENCE_API = process.env.NEMOCLAW_SWITCH_INFERENCE_API ?? "openai-completions"; const SWITCH_MOCK_ANTHROPIC = process.env.NEMOCLAW_SWITCH_MOCK_ANTHROPIC ?? "0"; const SWITCH_MOCK_PORT = parsePortEnv("NEMOCLAW_SWITCH_MOCK_PORT", 0); -const MOCK_BASELINE_API_KEY = "openclaw-switch-baseline-credential"; -const MOCK_BASELINE_MODEL = "openclaw-switch-baseline-model"; const TEST_TIMEOUT_MS = 75 * 60_000; const INSTALL_TIMEOUT_MS = 30 * 60_000; const COMMAND_TIMEOUT_MS = 120_000; @@ -129,27 +133,6 @@ interface MockAnthropicProvider { close(): Promise; } -interface BaselineInferenceConfig { - apiKey: string; - endpointUrl: string; - env: NodeJS.ProcessEnv; -} - -function mockBaselineInference(endpointUrl: string): BaselineInferenceConfig { - return { - apiKey: MOCK_BASELINE_API_KEY, - endpointUrl, - env: { - COMPATIBLE_API_KEY: MOCK_BASELINE_API_KEY, - NEMOCLAW_COMPAT_MODEL: MOCK_BASELINE_MODEL, - NEMOCLAW_ENDPOINT_URL: endpointUrl, - NEMOCLAW_MODEL: MOCK_BASELINE_MODEL, - NEMOCLAW_PREFERRED_API: "openai-completions", - NEMOCLAW_PROVIDER: "custom", - }, - }; -} - function expectMockBaselineAuthentication( baseline: Pick | undefined, ): void { @@ -747,12 +730,6 @@ function collectOpenClawAgentText(value: unknown, parts: string[], visited: Set< } } -function agentReplyContainsToken(reply: string, expected: string): boolean { - const normalizedReply = reply.replace(/\s+/gu, "").toUpperCase(); - const normalizedExpected = expected.replace(/\s+/gu, "").toUpperCase(); - return normalizedExpected.length > 0 && normalizedReply === normalizedExpected; -} - function parseOpenClawAgentText(raw: string): string { if (!raw.trim()) return ""; const parts: string[] = []; @@ -848,30 +825,10 @@ exit "$rc" ); } -test("openclaw-inference-switch agent reply matching tolerates wrapped PONG", () => { - expect(agentReplyContainsToken("P\nO N G", "PONG")).toBe(true); - expect(agentReplyContainsToken("wrapped: p o\nng", "PONG")).toBe(false); - expect(agentReplyContainsToken("the answer is PONG", "PONG")).toBe(false); - expect(agentReplyContainsToken("PONG because the route works", "PONG")).toBe(false); - expect(agentReplyContainsToken("PANG", "PONG")).toBe(false); - expect(agentReplyContainsToken("SPONGE", "PONG")).toBe(false); - expect(agentReplyContainsToken("pingpong", "PONG")).toBe(false); -}); - -test("openclaw mock-Anthropic switch uses an authenticated local baseline", () => { - expect(mockBaselineInference("http://127.0.0.1:34567/v1")).toEqual({ - apiKey: MOCK_BASELINE_API_KEY, - endpointUrl: "http://127.0.0.1:34567/v1", - env: { - COMPATIBLE_API_KEY: MOCK_BASELINE_API_KEY, - NEMOCLAW_COMPAT_MODEL: MOCK_BASELINE_MODEL, - NEMOCLAW_ENDPOINT_URL: "http://127.0.0.1:34567/v1", - NEMOCLAW_MODEL: MOCK_BASELINE_MODEL, - NEMOCLAW_PREFERRED_API: "openai-completions", - NEMOCLAW_PROVIDER: "custom", - }, - }); -}); +// The pure reply-matching and mock-baseline-config assertions that previously +// lived here as test(...) blocks (which only run under the opt-in live lane) +// are covered in the fast e2e-support project instead: +// test/e2e/support/openclaw-inference-switch-helpers.test.ts. function isExternalProviderValidationFailure(text: string): boolean { return ( diff --git a/test/e2e/live/skill-agent.test.ts b/test/e2e/live/skill-agent.test.ts index 52e1918f0ba..0b567a1a03d 100644 --- a/test/e2e/live/skill-agent.test.ts +++ b/test/e2e/live/skill-agent.test.ts @@ -3,7 +3,6 @@ import fs from "node:fs"; import path from "node:path"; -import { describe, it } from "vitest"; import { shellQuote } from "../../../src/lib/core/shell-quote"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { @@ -14,6 +13,13 @@ import { import { expect, test } from "../fixtures/e2e-test.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import { shouldRunLiveE2E } from "../fixtures/live-project-gate.ts"; +import { + agentSectionContainsToken, + isAgentVerificationFailClosed, + isExternalProviderValidationFailure, + shouldSkipExternalAgentVerificationFailure, + VERIFY_PHRASE, +} from "../support/skill-agent-classifiers.ts"; // Keep this as a direct live test: the the contract is skill fixture // injection into a real OpenClaw sandbox plus an agent turn that must read @@ -42,7 +48,6 @@ const VERIFY_SKILL_SCRIPT = path.join( const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-skill-agent"; validateSandboxName(SANDBOX_NAME); const SKILL_ID = "skill-smoke-fixture"; -const VERIFY_PHRASE = "SKILL_SMOKE_VERIFY_K9X2"; const ONBOARD_TIMEOUT_MS = 20 * 60_000; const AGENT_VERIFY_TIMEOUT_MS = 4 * 60_000; const MAX_ATTEMPTS = Number.parseInt(process.env.E2E_SKILL_AGENT_MAX_ATTEMPTS ?? "3", 10); @@ -59,54 +64,6 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -function isExternalAgentVerificationFlake(text: string): boolean { - // Only provider/model/transport timeout signatures are skippable, and only - // after the fixture is proven present. OpenClaw tool/runtime errors must fail - // this migration guard because the contract is that the real agent can read - // SKILL.md and return the token. This tolerance can be narrowed once the live - // provider/agent turn is consistently non-429/non-timeout in scheduled runs. - return /LLM idle timeout|request timed out|fetch timeout|model did not produce a response|ssh\/agent exit 124|exit 124|HTTP 429|\b429\b|rate[- ]?limit|quota|temporarily unavailable/i.test( - text, - ); -} - -function isAgentVerificationFailClosed(text: string): boolean { - // Preserve the existing helper's fail-closed ordering: a non-zero helper - // result that reports tool/security/runtime failure must not be turned into - // success just because the agent transcript also echoed the token. - return /SsrFBlockedError|Blocked hostname|Blocked: resolves to|transport error|provider error|ECONNREFUSED|EAI_AGAIN|gateway unavailable/i.test( - text, - ); -} - -function shouldSkipExternalAgentVerificationFailure( - text: string, - fixturePresent: boolean, -): boolean { - return ( - fixturePresent && !isAgentVerificationFailClosed(text) && isExternalAgentVerificationFlake(text) - ); -} - -function isExternalProviderValidationFailure(text: string): boolean { - // Onboarding can fail before sandbox creation when the external NVIDIA - // endpoint validation is rate-limited or unavailable. Treat only those - // live-service states as inconclusive; repo-local onboarding errors still - // fail. This can be narrowed when endpoint validation stops producing - // intermittent 429/timeout failures in scheduled live runs. - return ( - /NVIDIA Endpoints endpoint validation failed/i.test(text) && - /HTTP 429|rate limit|quota|temporarily unavailable|timed out|timeout/i.test(text) - ); -} - -function agentSectionContainsToken(agentOutput: string): boolean { - const match = agentOutput.match(/--- agent stdout\/stderr[\s\S]*?--- end ---/); - if (!match) return false; - const collapsed = match[0].replace(/[\n\r`"']/g, "").toLowerCase(); - return collapsed.includes(VERIFY_PHRASE.toLowerCase()); -} - function buildVerifySkillFixtureScript(): string { // OpenShell rejects newline-bearing command args, so keep this readable as // discrete clauses while emitting a single-line `sh -lc` script. @@ -150,46 +107,6 @@ async function ignoreCleanupError(run: () => Promise): Promise { } } -describe("skill-agent live test local classifiers", () => { - it("does not treat helper fail-closed output as a skippable provider flake", () => { - const output = `--- agent stdout/stderr\nSsrFBlockedError\n${VERIFY_PHRASE}\n--- end ---`; - - expect(isAgentVerificationFailClosed(output)).toBe(true); - expect(shouldSkipExternalAgentVerificationFailure(output, true)).toBe(false); - }); - - it("skips only timeout-like agent verification failures after fixture presence is proven", () => { - const timeoutOutput = `--- agent stdout/stderr\nLLM idle timeout\n--- end ---`; - - expect(shouldSkipExternalAgentVerificationFailure(timeoutOutput, false)).toBe(false); - expect(shouldSkipExternalAgentVerificationFailure(timeoutOutput, true)).toBe(true); - expect(shouldSkipExternalAgentVerificationFailure("require is not defined", true)).toBe(false); - expect(shouldSkipExternalAgentVerificationFailure("HTTP 429 rate limit", true)).toBe(true); - expect( - shouldSkipExternalAgentVerificationFailure("SsrFBlockedError plus request timed out", true), - ).toBe(false); - }); - - it("skips only NVIDIA endpoint validation outages during onboarding", () => { - expect( - isExternalProviderValidationFailure( - "NVIDIA Endpoints endpoint validation failed.\nChat Completions API validation returned HTTP 429", - ), - ).toBe(true); - expect(isExternalProviderValidationFailure("local docker preflight timed out")).toBe(false); - expect( - isExternalProviderValidationFailure("NVIDIA Endpoints endpoint validation failed."), - ).toBe(false); - }); - - it("matches the token only inside the delimited agent section", () => { - expect(agentSectionContainsToken(`helper echoed ${VERIFY_PHRASE}`)).toBe(false); - expect( - agentSectionContainsToken(`--- agent stdout/stderr\n\`${VERIFY_PHRASE}\`\n--- end ---`), - ).toBe(true); - }); -}); - const runSkillAgentTest = shouldRunLiveE2E() ? test : test.skip; runSkillAgentTest( diff --git a/test/e2e/support/common-egress-agent-helpers.test.ts b/test/e2e/support/common-egress-agent-helpers.test.ts new file mode 100644 index 00000000000..4f33e6141af --- /dev/null +++ b/test/e2e/support/common-egress-agent-helpers.test.ts @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + agentReplyContainsToken, + classifyPreContractProviderValidationSkip, + parseChatContent, + parseOpenClawAgentText, +} from "../live/common-egress-agent-helpers.ts"; + +describe("common-egress agent parsing and classification helpers", () => { + it("OpenClaw JSON parser accepts framed agent payloads", () => { + expect( + parseOpenClawAgentText( + JSON.stringify({ payloads: [{ text: "noise" }, { text: "WEATHER_AGENT_OK" }] }), + ), + ).toContain("WEATHER_AGENT_OK"); + expect( + parseOpenClawAgentText( + JSON.stringify({ result: { payloads: [{ text: "REFERENCE_AGENT_OK" }] } }), + ), + ).toContain("REFERENCE_AGENT_OK"); + expect( + parseOpenClawAgentText( + `openclaw log line\n${JSON.stringify({ + result: { payloads: [{ text: "HERMES_REFERENCE_AGENT_OK" }] }, + })}\n`, + ), + ).toContain("HERMES_REFERENCE_AGENT_OK"); + }); + + it("Hermes response parser reads message content", () => { + expect( + parseChatContent( + JSON.stringify({ choices: [{ message: { content: "HERMES_REFERENCE_AGENT_OK" } }] }), + ), + ).toBe("HERMES_REFERENCE_AGENT_OK"); + }); + + it("expected-token matching ignores model line breaks", () => { + expect(agentReplyContainsToken("REFER\nENCE_AGENT_OK", "REFERENCE_AGENT_OK")).toBe(true); + expect( + agentReplyContainsToken("HERMES_REFERENCE\n_AGENT_OK", "HERMES_REFERENCE_AGENT_OK"), + ).toBe(true); + }); + + it("classifies pre-contract provider validation skips", () => { + expect( + classifyPreContractProviderValidationSkip({ + stdout: "", + stderr: + "NVIDIA Endpoints endpoint validation failed.\nChat Completions API validation returned HTTP 429", + }), + ).toMatchObject({ + http429ProviderValidationFailure: true, + matches: true, + }); + + const originalGithubActions = process.env.GITHUB_ACTIONS; + const restoreGithubActions = () => { + delete process.env.GITHUB_ACTIONS; + Object.assign( + process.env, + originalGithubActions === undefined ? {} : { GITHUB_ACTIONS: originalGithubActions }, + ); + }; + try { + process.env.GITHUB_ACTIONS = "true"; + expect( + classifyPreContractProviderValidationSkip({ + stdout: "", + stderr: + "NVIDIA Endpoints endpoint validation failed.\nValidation details were omitted to avoid exposing credentials.", + }), + ).toMatchObject({ + matches: true, + sanitizedEndpointValidationFailure: true, + }); + } finally { + restoreGithubActions(); + } + + expect( + classifyPreContractProviderValidationSkip({ + stdout: "", + stderr: + "NVIDIA Endpoints endpoint validation failed.\ninvalid NVIDIA_INFERENCE_API_KEY credential", + }), + ).toMatchObject({ matches: false }); + }); +}); diff --git a/test/e2e/support/messaging-endpoint-classifiers.test.ts b/test/e2e/support/messaging-endpoint-classifiers.test.ts new file mode 100644 index 00000000000..96ae27ef9c1 --- /dev/null +++ b/test/e2e/support/messaging-endpoint-classifiers.test.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { parseOpenClawAgentText } from "../live/messaging-compatible-endpoint-helpers.ts"; +import { COMPAT_AGENT_PROMPT, COMPAT_AGENT_REPLY } from "./messaging-endpoint-classifiers.ts"; + +describe("messaging-compatible-endpoint live test local classifiers", () => { + it("does not satisfy the agent reply assertion with echoed prompt text", () => { + expect(COMPAT_AGENT_PROMPT).not.toContain(COMPAT_AGENT_REPLY); + expect( + parseOpenClawAgentText(JSON.stringify({ result: { content: COMPAT_AGENT_PROMPT } })), + ).not.toContain(COMPAT_AGENT_REPLY); + expect( + parseOpenClawAgentText(JSON.stringify({ result: { content: COMPAT_AGENT_REPLY } })), + ).toContain(COMPAT_AGENT_REPLY); + }); +}); diff --git a/test/e2e/support/messaging-endpoint-classifiers.ts b/test/e2e/support/messaging-endpoint-classifiers.ts new file mode 100644 index 00000000000..7f6bc5f5103 --- /dev/null +++ b/test/e2e/support/messaging-endpoint-classifiers.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Pure reply-assertion constants shared by the messaging-compatible-endpoint +// live E2E target and its PR-collected unit tests. Extracting the token +// constants lets the fast e2e-support project verify that the agent reply +// assertion cannot be satisfied by echoed prompt text without gating on +// NEMOCLAW_RUN_LIVE_E2E=1. + +// Token the mock compatible endpoint returns and the agent turn must echo back. +export const COMPAT_AGENT_REPLY = "COMPAT_MOCK_ROUTE_5098_OK"; +export const COMPAT_AGENT_PROMPT = + "Call the configured model and report the compatible endpoint route token."; diff --git a/test/e2e/support/openclaw-inference-switch-helpers.test.ts b/test/e2e/support/openclaw-inference-switch-helpers.test.ts new file mode 100644 index 00000000000..c8f9ca0d91a --- /dev/null +++ b/test/e2e/support/openclaw-inference-switch-helpers.test.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + agentReplyContainsToken, + MOCK_BASELINE_API_KEY, + MOCK_BASELINE_MODEL, + mockBaselineInference, +} from "../live/openclaw-inference-switch-helpers.ts"; + +describe("openclaw-inference-switch agent reply matching", () => { + it("tolerates wrapped PONG", () => { + expect(agentReplyContainsToken("P\nO N G", "PONG")).toBe(true); + expect(agentReplyContainsToken("wrapped: p o\nng", "PONG")).toBe(false); + expect(agentReplyContainsToken("the answer is PONG", "PONG")).toBe(false); + expect(agentReplyContainsToken("PONG because the route works", "PONG")).toBe(false); + expect(agentReplyContainsToken("PANG", "PONG")).toBe(false); + expect(agentReplyContainsToken("SPONGE", "PONG")).toBe(false); + expect(agentReplyContainsToken("pingpong", "PONG")).toBe(false); + }); +}); + +describe("openclaw-inference-switch mock-Anthropic baseline", () => { + it("uses an authenticated local baseline with the compatible env wiring", () => { + expect(mockBaselineInference("http://127.0.0.1:34567/v1")).toEqual({ + apiKey: MOCK_BASELINE_API_KEY, + endpointUrl: "http://127.0.0.1:34567/v1", + env: { + COMPATIBLE_API_KEY: MOCK_BASELINE_API_KEY, + NEMOCLAW_COMPAT_MODEL: MOCK_BASELINE_MODEL, + NEMOCLAW_ENDPOINT_URL: "http://127.0.0.1:34567/v1", + NEMOCLAW_MODEL: MOCK_BASELINE_MODEL, + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + }, + }); + }); + + it("threads the endpoint URL into both the config and the env", () => { + const baseline = mockBaselineInference("http://10.0.0.5:9000/v1"); + expect(baseline.endpointUrl).toBe("http://10.0.0.5:9000/v1"); + expect(baseline.env.NEMOCLAW_ENDPOINT_URL).toBe("http://10.0.0.5:9000/v1"); + }); +}); diff --git a/test/e2e/support/skill-agent-classifiers.test.ts b/test/e2e/support/skill-agent-classifiers.test.ts new file mode 100644 index 00000000000..b38c5a1ff56 --- /dev/null +++ b/test/e2e/support/skill-agent-classifiers.test.ts @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + agentSectionContainsToken, + isAgentVerificationFailClosed, + isExternalProviderValidationFailure, + shouldSkipExternalAgentVerificationFailure, + VERIFY_PHRASE, +} from "./skill-agent-classifiers.ts"; + +describe("skill-agent live test local classifiers", () => { + it("does not treat helper fail-closed output as a skippable provider flake", () => { + const output = `--- agent stdout/stderr\nSsrFBlockedError\n${VERIFY_PHRASE}\n--- end ---`; + + expect(isAgentVerificationFailClosed(output)).toBe(true); + expect(shouldSkipExternalAgentVerificationFailure(output, true)).toBe(false); + }); + + it("skips only timeout-like agent verification failures after fixture presence is proven", () => { + const timeoutOutput = `--- agent stdout/stderr\nLLM idle timeout\n--- end ---`; + + expect(shouldSkipExternalAgentVerificationFailure(timeoutOutput, false)).toBe(false); + expect(shouldSkipExternalAgentVerificationFailure(timeoutOutput, true)).toBe(true); + expect(shouldSkipExternalAgentVerificationFailure("require is not defined", true)).toBe(false); + expect(shouldSkipExternalAgentVerificationFailure("HTTP 429 rate limit", true)).toBe(true); + expect( + shouldSkipExternalAgentVerificationFailure("SsrFBlockedError plus request timed out", true), + ).toBe(false); + }); + + it("skips only NVIDIA endpoint validation outages during onboarding", () => { + expect( + isExternalProviderValidationFailure( + "NVIDIA Endpoints endpoint validation failed.\nChat Completions API validation returned HTTP 429", + ), + ).toBe(true); + expect(isExternalProviderValidationFailure("local docker preflight timed out")).toBe(false); + expect( + isExternalProviderValidationFailure("NVIDIA Endpoints endpoint validation failed."), + ).toBe(false); + }); + + it("matches the token only inside the delimited agent section", () => { + expect(agentSectionContainsToken(`helper echoed ${VERIFY_PHRASE}`)).toBe(false); + expect( + agentSectionContainsToken(`--- agent stdout/stderr\n\`${VERIFY_PHRASE}\`\n--- end ---`), + ).toBe(true); + }); +}); diff --git a/test/e2e/support/skill-agent-classifiers.ts b/test/e2e/support/skill-agent-classifiers.ts new file mode 100644 index 00000000000..415b3931f63 --- /dev/null +++ b/test/e2e/support/skill-agent-classifiers.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Pure predicate helpers shared by the skill-agent live E2E target and its +// PR-collected unit tests. Keeping them here lets the fast e2e-support project +// exercise the classification logic without gating on NEMOCLAW_RUN_LIVE_E2E=1. + +// Token the injected skill fixture must echo back through the agent transcript. +export const VERIFY_PHRASE = "SKILL_SMOKE_VERIFY_K9X2"; + +export function isExternalAgentVerificationFlake(text: string): boolean { + // Only provider/model/transport timeout signatures are skippable, and only + // after the fixture is proven present. OpenClaw tool/runtime errors must fail + // this migration guard because the contract is that the real agent can read + // SKILL.md and return the token. This tolerance can be narrowed once the live + // provider/agent turn is consistently non-429/non-timeout in scheduled runs. + return /LLM idle timeout|request timed out|fetch timeout|model did not produce a response|ssh\/agent exit 124|exit 124|HTTP 429|\b429\b|rate[- ]?limit|quota|temporarily unavailable/i.test( + text, + ); +} + +export function isAgentVerificationFailClosed(text: string): boolean { + // Preserve the existing helper's fail-closed ordering: a non-zero helper + // result that reports tool/security/runtime failure must not be turned into + // success just because the agent transcript also echoed the token. + return /SsrFBlockedError|Blocked hostname|Blocked: resolves to|transport error|provider error|ECONNREFUSED|EAI_AGAIN|gateway unavailable/i.test( + text, + ); +} + +export function shouldSkipExternalAgentVerificationFailure( + text: string, + fixturePresent: boolean, +): boolean { + return ( + fixturePresent && !isAgentVerificationFailClosed(text) && isExternalAgentVerificationFlake(text) + ); +} + +export function isExternalProviderValidationFailure(text: string): boolean { + // Onboarding can fail before sandbox creation when the external NVIDIA + // endpoint validation is rate-limited or unavailable. Treat only those + // live-service states as inconclusive; repo-local onboarding errors still + // fail. This can be narrowed when endpoint validation stops producing + // intermittent 429/timeout failures in scheduled live runs. + return ( + /NVIDIA Endpoints endpoint validation failed/i.test(text) && + /HTTP 429|rate limit|quota|temporarily unavailable|timed out|timeout/i.test(text) + ); +} + +export function agentSectionContainsToken( + agentOutput: string, + verifyPhrase: string = VERIFY_PHRASE, +): boolean { + const match = agentOutput.match(/--- agent stdout\/stderr[\s\S]*?--- end ---/); + if (!match) return false; + const collapsed = match[0].replace(/[\n\r`"']/g, "").toLowerCase(); + return collapsed.includes(verifyPhrase.toLowerCase()); +} diff --git a/test/hermes-env-secret-boundary-hardening.test.ts b/test/hermes-env-secret-boundary-hardening.test.ts index 4c7693fdf71..b02223e3fd8 100644 --- a/test/hermes-env-secret-boundary-hardening.test.ts +++ b/test/hermes-env-secret-boundary-hardening.test.ts @@ -51,7 +51,12 @@ function runStartEnvValidation(hermesDir: string) { [ "#!/usr/bin/env bash", "set -u", - "_HERMES_BOUNDARY_TIMEOUT=()", + // A harmless single-element no-op prefix. It must not be an empty array: + // macOS bash 3.2 treats "${empty[@]}" as an unbound variable under + // `set -u`, aborting the harness before the validator runs. It also must + // not be `env --`: macOS/BSD `env(1)` does not support `--`. The + // `command` builtin execs the validator unchanged on every platform. + "_HERMES_BOUNDARY_TIMEOUT=(command)", '_HERMES_PYTHON="$(command -v python3)"', `_HERMES_BOUNDARY_VALIDATOR=${JSON.stringify(VALIDATOR)}`, `HERMES_DIR=${JSON.stringify(hermesDir)}`, @@ -70,6 +75,44 @@ function runStartEnvValidation(hermesDir: string) { } } +function runRuntimeEnvValidation(envOverrides: Record) { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const runDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-env-check-")); + const script = path.join(runDir, "run.sh"); + try { + fs.writeFileSync( + script, + [ + "#!/usr/bin/env bash", + "set -u", + // A harmless single-element no-op prefix. It must not be an empty array: + // macOS bash 3.2 treats "${empty[@]}" as an unbound variable under + // `set -u`, aborting the harness before the validator runs. It also must + // not be `env --`: macOS/BSD `env(1)` does not support `--`. The + // `command` builtin execs the validator unchanged on every platform. + "_HERMES_BOUNDARY_TIMEOUT=(command)", + '_HERMES_PYTHON="$(command -v python3)"', + `_HERMES_BOUNDARY_VALIDATOR=${JSON.stringify(VALIDATOR)}`, + extractShellFunction(source, "validate_hermes_runtime_env_secret_boundary"), + "validate_hermes_runtime_env_secret_boundary", + ].join("\n"), + { mode: 0o700 }, + ); + return spawnSync("bash", [script], { + encoding: "utf-8", + timeout: 5000, + env: { + HOME: os.tmpdir(), + PATH: process.env.PATH ?? "", + _HERMES_BOUNDARY_VALIDATOR: VALIDATOR, + ...envOverrides, + }, + }); + } finally { + fs.rmSync(runDir, { recursive: true, force: true }); + } +} + describe("Hermes env secret-boundary resource limits", () => { it("accepts the normal 0640 mutable env-file mode", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-env-mode-")); @@ -386,3 +429,67 @@ wait "$child" } }); }); + +describe("Hermes env secret-boundary value-shape discriminator", () => { + it("accepts the same secret-shaped key once its value is an openshell resolver placeholder", () => { + // The reject path aborts on DEVTEST_API_TOKEN=. Pin the other side of + // the boundary: the identical secret-shaped key flips to accepted solely + // because the value is a resolver reference, so the discriminator is the + // value shape (raw vs. placeholder), not the key name. + const rawToken = "SENTINEL_RAW_SECRET_VALUE"; + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-env-shape-file-accept-")); + const hermes = path.join(root, ".hermes"); + fs.mkdirSync(hermes, { recursive: true }); + fs.writeFileSync( + path.join(hermes, ".env"), + "DEVTEST_API_TOKEN=openshell:resolve:env:DEVTEST_API_TOKEN\n", + { mode: 0o600 }, + ); + try { + const result = runStartEnvValidation(hermes); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stderr).not.toContain(rawToken); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("accepts a non-secret-shaped key carrying a raw value", () => { + // A key that does not match the secret pattern may hold a literal value; + // the boundary must not abort on ordinary config that merely looks opaque. + const rawValue = "SENTINEL_RAW_SECRET_VALUE"; + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-env-shape-file-nonsecret-")); + const hermes = path.join(root, ".hermes"); + fs.mkdirSync(hermes, { recursive: true }); + fs.writeFileSync(path.join(hermes, ".env"), `DEVTEST_ENDPOINT=${rawValue}\n`, { mode: 0o600 }); + try { + const result = runStartEnvValidation(hermes); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("accepts the same secret-shaped process env key once its value is a resolver placeholder", () => { + const rawToken = "SENTINEL_RAW_SECRET_VALUE"; + const result = runRuntimeEnvValidation({ + DEVTEST_API_TOKEN: "openshell:resolve:env:DEVTEST_API_TOKEN", + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stderr).not.toContain(rawToken); + }); + + it("accepts a non-secret-shaped process env key carrying a raw value", () => { + const rawValue = "SENTINEL_RAW_SECRET_VALUE"; + const result = runRuntimeEnvValidation({ + DEVTEST_ENDPOINT: rawValue, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + }); +}); diff --git a/test/hermes-gateway-pid-cleanup-helpers.ts b/test/hermes-gateway-pid-cleanup-helpers.ts new file mode 100644 index 00000000000..31cdfc42c34 --- /dev/null +++ b/test/hermes-gateway-pid-cleanup-helpers.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Test harness helpers for hermes-gateway-pid-cleanup.test.ts. The shell- +// function extraction + invocation branching lives here (not in the *.test.ts) +// so the test body stays linear. + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const START_SCRIPT = path.join(import.meta.dirname, "..", "agents", "hermes", "start.sh"); + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function extractShellFunctionFromSource(src: string, name: string): string { + const escapedName = escapeRegExp(name); + const match = src.match(new RegExp(`${escapedName}\\(\\) \\{([\\s\\S]*?)^\\}`, "m")); + if (!match) { + throw new Error(`Expected ${name} in agents/hermes/start.sh`); + } + return `${name}() {${match[1]}\n}`; +} + +/** + * Extract remove_stale_gateway_file and run it against `pidPath` inside a + * throwaway temp dir. Returns the spawn result plus the temp root so callers + * can assert on the resulting on-disk shape. + */ +export function runRemoveStale( + seed: (tmp: string, pidPath: string) => void, + label = "legacy PID file", +): { status: number | null; stderr: string; tmp: string; pidPath: string } { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const fn = extractShellFunctionFromSource(src, "remove_stale_gateway_file"); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "hermes-gw-pid-cleanup-")); + const pidPath = path.join(tmp, "gateway.pid"); + seed(tmp, pidPath); + + const script = [ + "set -euo pipefail", + fn, + `remove_stale_gateway_file ${JSON.stringify(pidPath)} ${JSON.stringify(label)}`, + ].join("\n"); + + const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); + return { status: result.status, stderr: result.stderr, tmp, pidPath }; +} diff --git a/test/hermes-gateway-pid-cleanup.test.ts b/test/hermes-gateway-pid-cleanup.test.ts new file mode 100644 index 00000000000..0faad7cd775 --- /dev/null +++ b/test/hermes-gateway-pid-cleanup.test.ts @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Mocked shell-unit coverage for the Hermes gateway-PID-file cleanup contract. +// remove_stale_gateway_file() is the seam guarding the root-owned gateway.pid +// path: a stale regular file OR a symlink at the PID path must be removed +// (never symlink-followed) so the resulting gateway.pid is always a regular +// file, never a symlink. Previously this was only proven by the live +// test/e2e/live/hermes-root-entrypoint-smoke.test.ts legacy-migration case. + +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +import { runRemoveStale } from "./hermes-gateway-pid-cleanup-helpers.ts"; + +describe("Hermes remove_stale_gateway_file cleanup (legacy gateway.pid)", () => { + it("removes a symlink at the PID path without following it, leaving no symlink target damage", () => { + // A symlink pointing at a real target file must be removed itself; the + // target must remain untouched (refuse to follow the link). + let targetPath = ""; + const { status, stderr, tmp, pidPath } = runRemoveStale((tmpDir, pid) => { + targetPath = path.join(tmpDir, "real-target"); + fs.writeFileSync(targetPath, "gateway target contents\n"); + fs.symlinkSync(targetPath, pid); + }); + + try { + expect(status).toBe(0); + expect(stderr).toContain("Removing unsafe stale Hermes legacy PID file symlink"); + // The symlink at the PID path is gone. + expect(fs.existsSync(pidPath)).toBe(false); + // The symlink was NOT followed: its target file is intact. + expect(fs.existsSync(targetPath)).toBe(true); + expect(fs.readFileSync(targetPath, "utf-8")).toBe("gateway target contents\n"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("removes a stale regular file at the PID path", () => { + const { status, stderr, tmp, pidPath } = runRemoveStale((_tmpDir, pid) => { + fs.writeFileSync(pid, "12345 987654\n"); + }); + + try { + expect(status).toBe(0); + expect(stderr).toContain("Removing stale Hermes legacy PID file"); + expect(fs.existsSync(pidPath)).toBe(false); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("is a no-op when nothing exists at the PID path (fresh start)", () => { + const { status, stderr, tmp, pidPath } = runRemoveStale(() => { + // Seed nothing: pidPath does not exist. + }); + + try { + expect(status).toBe(0); + expect(stderr).not.toContain("Removing"); + expect(fs.existsSync(pidPath)).toBe(false); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("removes a dangling symlink (broken legacy link) so a regular file can replace it", () => { + // A symlink whose target no longer exists is still unsafe at the root-owned + // PID path; it must be removed so a later writer creates a regular file. + const { status, stderr, tmp, pidPath } = runRemoveStale((tmpDir, pid) => { + fs.symlinkSync(path.join(tmpDir, "does-not-exist"), pid); + }); + + try { + expect(status).toBe(0); + expect(stderr).toContain("Removing unsafe stale Hermes legacy PID file symlink"); + // lstat-based existence: the dangling symlink itself is gone. + expect(fs.existsSync(pidPath)).toBe(false); + let lstatFailed = false; + try { + fs.lstatSync(pidPath); + } catch { + lstatFailed = true; + } + expect(lstatFailed).toBe(true); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); diff --git a/test/hermes-start.test.ts b/test/hermes-start.test.ts index e8686ae666b..2eac9051dc2 100644 --- a/test/hermes-start.test.ts +++ b/test/hermes-start.test.ts @@ -171,7 +171,12 @@ function runHermesEnvSecretBoundary(opts: { envFile?: string; symlinkEnvFile?: b [ "#!/usr/bin/env bash", "set -euo pipefail", - '_HERMES_BOUNDARY_TIMEOUT=(); _HERMES_PYTHON="$(command -v python3)"', + // A harmless single-element no-op prefix. It must not be an empty array: + // macOS bash 3.2 treats "${empty[@]}" as an unbound variable under + // `set -u`, aborting the harness before the validator runs. It also must + // not be `env --`: macOS/BSD `env(1)` does not support `--`. The + // `command` builtin execs the validator unchanged on every platform. + '_HERMES_BOUNDARY_TIMEOUT=(command); _HERMES_PYTHON="$(command -v python3)"', extractShellFunctionFromSource(src, "validate_hermes_env_secret_boundary"), `HERMES_DIR=${shellQuote(hermesHome)}`, `_HERMES_BOUNDARY_VALIDATOR=${shellQuote(SECRET_BOUNDARY_VALIDATOR_SCRIPT)}`, @@ -200,7 +205,12 @@ function runHermesRuntimeEnvSecretBoundary(envOverrides: Record) [ "#!/usr/bin/env bash", "set -euo pipefail", - '_HERMES_BOUNDARY_TIMEOUT=(); _HERMES_PYTHON="$(command -v python3)"', + // A harmless single-element no-op prefix. It must not be an empty array: + // macOS bash 3.2 treats "${empty[@]}" as an unbound variable under + // `set -u`, aborting the harness before the validator runs. It also must + // not be `env --`: macOS/BSD `env(1)` does not support `--`. The + // `command` builtin execs the validator unchanged on every platform. + '_HERMES_BOUNDARY_TIMEOUT=(command); _HERMES_PYTHON="$(command -v python3)"', extractShellFunctionFromSource(src, "validate_hermes_runtime_env_secret_boundary"), `_HERMES_BOUNDARY_VALIDATOR=${shellQuote(SECRET_BOUNDARY_VALIDATOR_SCRIPT)}`, "validate_hermes_runtime_env_secret_boundary", diff --git a/test/nemoclaw-start-extra-placeholder-breadcrumb-helpers.ts b/test/nemoclaw-start-extra-placeholder-breadcrumb-helpers.ts new file mode 100644 index 00000000000..ad84fb8044e --- /dev/null +++ b/test/nemoclaw-start-extra-placeholder-breadcrumb-helpers.ts @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Test harness helpers for nemoclaw-start-extra-placeholder-breadcrumb.test.ts. +// The heredoc-aware shell-function extractor and the refresh invocation wrapper +// (both branching) live here so the test body stays linear. + +import { type SpawnSyncReturns, spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; + +export const START_SCRIPT = path.join(import.meta.dirname, "..", "scripts", "nemoclaw-start.sh"); + +// Heredoc-aware extractor. The reconcile harness's naive /^}/m regex stops at +// the first column-0 "}", which for refresh_openclaw_provider_placeholders is +// the closing brace of a Python dict comprehension inside a <<'PY…' heredoc, +// not the function's real close. Skip heredoc bodies so we capture the whole +// function. +export function extractShellFunction(src: string, name: string): string { + const lines = src.split("\n"); + const start = lines.findIndex((line) => line.startsWith(`${name}() {`)); + if (start < 0) throw new Error(`Expected ${name} in scripts/nemoclaw-start.sh`); + let heredocTerminator: string | null = null; + for (let i = start + 1; i < lines.length; i++) { + const line = lines[i]; + if (heredocTerminator !== null) { + if (line === heredocTerminator) heredocTerminator = null; + continue; + } + const opener = line.match(/<<-?\s*'?([A-Za-z_][A-Za-z0-9_]*)'?/); + if (opener) { + heredocTerminator = opener[1]; + continue; + } + if (line === "}") return lines.slice(start, i + 1).join("\n"); + } + throw new Error(`Expected a top-level close for ${name} in scripts/nemoclaw-start.sh`); +} + +export interface RunResult { + result: SpawnSyncReturns; + // Arbitrary caller-shaped openclaw.json indexed directly by tests + // (config.channels.telegram…), matching the original inline helper's typing. + // biome noExplicitAny is not enforced under test/, so no suppression is needed. + config: any; +} + +export function runRefresh(config: unknown, env: Record = {}): RunResult { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-extra-placeholder-")); + const openclawDir = path.join(root, ".openclaw"); + fs.mkdirSync(openclawDir, { recursive: true }); + const configPath = path.join(openclawDir, "openclaw.json"); + const hashPath = path.join(openclawDir, ".config-hash"); + fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`); + fs.writeFileSync(hashPath, "oldhash\n"); + + const fn = extractShellFunction(src, "refresh_openclaw_provider_placeholders").replaceAll( + "/sandbox/.openclaw", + openclawDir, + ); + // Stub the config-mutability guards and the dir-owner probe so the helper + // runs on a mutable temp dir without touching real sandbox ownership. This + // isolates the extras-validation + placeholder-rewrite path under test. + const wrapper = [ + "#!/usr/bin/env bash", + "set -eu", + "openclaw_config_dir_owner() { echo sandbox; }", + "prepare_openclaw_config_for_write() { :; }", + "restore_openclaw_config_after_write() { :; }", + fn, + "refresh_openclaw_provider_placeholders", + ].join("\n"); + const script = path.join(root, "run.sh"); + fs.writeFileSync(script, wrapper, { mode: 0o700 }); + try { + const result = spawnSync("bash", [script], { + encoding: "utf-8", + env: { PATH: process.env.PATH || "", ...env }, + timeout: 5000, + }); + const updated = JSON.parse(fs.readFileSync(configPath, "utf-8")); + return { result, config: updated }; + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +} + +// Mirror the messaging-runtime plan the entrypoint forwards so the in- +// container parser discovers TELEGRAM_BOT_TOKEN as a canonical provider +// envKey; per-profile TELEGRAM_BOT_TOKEN_AGENT_* names then read as valid +// extensions rather than colliding with a canonical base key. +export function placeholderPlan(envKeys: string[]): string { + return Buffer.from( + JSON.stringify({ + credentialBindings: envKeys.map((envKey) => ({ providerEnvKey: envKey })), + }), + ).toString("base64"); +} diff --git a/test/nemoclaw-start-extra-placeholder-breadcrumb.test.ts b/test/nemoclaw-start-extra-placeholder-breadcrumb.test.ts new file mode 100644 index 00000000000..84180ffd2e9 --- /dev/null +++ b/test/nemoclaw-start-extra-placeholder-breadcrumb.test.ts @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + placeholderPlan, + runRefresh, +} from "./nemoclaw-start-extra-placeholder-breadcrumb-helpers.ts"; + +// The extra-placeholder canonicalization + accepted-keys breadcrumb contract is +// asserted end-to-end only in the live messaging-providers E2E (cases X4a/X4b +// on the canonical resolve placeholders and X5 on the accepted-extras +// breadcrumb). That lane runs on an ephemeral Brev instance and never gates PR +// CI, so this mocked shell-unit pins the same three properties against the real +// `refresh_openclaw_provider_placeholders` body extracted from +// scripts/nemoclaw-start.sh: +// X4a/X4b — each accepted extra key becomes a canonical +// openshell:resolve:env: placeholder, and distinct extra keys resolve +// to distinct placeholders. +// X5 — the startup breadcrumb "[config] NEMOCLAW_EXTRA_PLACEHOLDER_KEYS +// accepted N entry(ies): …" lists only the accepted keys and omits any +// refused key (e.g. GITHUB_TOKEN). +// The host-side TS mirror (src/lib/onboard/extra-placeholder-keys.ts) is unit- +// tested separately; the openshell:resolve:env: literal and the +// accepted-keys summary string live solely in the shell function, so they need +// a shell-unit here. (#4251) + +describe("extra-placeholder canonicalization + accepted-extras breadcrumb (X4a/X4b/X5)", () => { + it("resolves distinct accepted extra keys to distinct canonical openshell:resolve:env placeholders (X4a/X4b)", () => { + // openclaw.json carries the baked canonical placeholders for two per-profile + // extension keys; the runtime env stages a canonical (non-revision) + // OpenShell resolve placeholder for each. Both must be accepted and each + // profile must end up carrying its own canonical openshell:resolve:env: + // placeholder — the X4a/X4b assertions. + const canonicalA = "openshell:resolve:env:TELEGRAM_BOT_TOKEN_AGENT_A"; + const canonicalB = "openshell:resolve:env:TELEGRAM_BOT_TOKEN_AGENT_B"; + const run = runRefresh( + { + channels: { + telegram: { + accounts: { + a: { botToken: canonicalA }, + b: { botToken: canonicalB }, + }, + }, + }, + }, + { + NEMOCLAW_MESSAGING_PLAN_B64: placeholderPlan(["TELEGRAM_BOT_TOKEN"]), + NEMOCLAW_EXTRA_PLACEHOLDER_KEYS: "TELEGRAM_BOT_TOKEN_AGENT_A TELEGRAM_BOT_TOKEN_AGENT_B", + TELEGRAM_BOT_TOKEN_AGENT_A: canonicalA, + TELEGRAM_BOT_TOKEN_AGENT_B: canonicalB, + }, + ); + + expect(run.result.status, run.result.stderr).toBe(0); + const tokenA = run.config.channels.telegram.accounts.a.botToken; + const tokenB = run.config.channels.telegram.accounts.b.botToken; + // X4a / X4b: each accepted extra key is a canonical OpenShell resolve + // placeholder for exactly its own env key. + expect(tokenA).toBe(canonicalA); + expect(tokenB).toBe(canonicalB); + expect(tokenA.startsWith("openshell:resolve:env:")).toBe(true); + expect(tokenB.startsWith("openshell:resolve:env:")).toBe(true); + // X4b: distinct extension keys must resolve to distinct placeholders — the + // grammar-aware exact-token rewrite must never collapse AGENT_B onto + // AGENT_A's placeholder. + expect(tokenA).not.toBe(tokenB); + }); + + it("names accepted extra keys in the breadcrumb and omits a co-submitted refused GITHUB_TOKEN (X5)", () => { + // The operator submits one accepted per-profile extension plus a refused + // arbitrary host secret (GITHUB_TOKEN) in the same control env. The X5 + // breadcrumb must list the accepted key and MUST NOT name the refused key, + // proving a refused host secret cannot ride the accepted-extras summary into + // the sandbox provider gateway. + const run = runRefresh( + { + channels: { + telegram: { + accounts: { + a: { botToken: "openshell:resolve:env:TELEGRAM_BOT_TOKEN_AGENT_A" }, + }, + }, + }, + }, + { + NEMOCLAW_MESSAGING_PLAN_B64: placeholderPlan(["TELEGRAM_BOT_TOKEN"]), + NEMOCLAW_EXTRA_PLACEHOLDER_KEYS: "GITHUB_TOKEN TELEGRAM_BOT_TOKEN_AGENT_A", + GITHUB_TOKEN: "ghp-host-secret-would-leak", + TELEGRAM_BOT_TOKEN_AGENT_A: "openshell:resolve:env:TELEGRAM_BOT_TOKEN_AGENT_A", + }, + ); + + expect(run.result.status, run.result.stderr).toBe(0); + const breadcrumb = run.result.stderr + .split("\n") + .find((line) => line.includes("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS accepted")); + expect(breadcrumb, run.result.stderr).toBeDefined(); + // X5: exactly one accepted entry, named, and the refused key absent from the + // accepted summary line. + expect(breadcrumb).toMatch( + /^\[config\] NEMOCLAW_EXTRA_PLACEHOLDER_KEYS accepted 1 entry\(ies\): TELEGRAM_BOT_TOKEN_AGENT_A$/, + ); + expect(breadcrumb).not.toContain("GITHUB_TOKEN"); + // The refused key is reported only on its own ignore line, never as an + // accepted entry, and its staged value never leaks into any output. + expect(run.result.stderr).toContain( + "[config] Ignoring NEMOCLAW_EXTRA_PLACEHOLDER_KEYS entry 'GITHUB_TOKEN' — must extend a discovered provider envKey such as TELEGRAM_BOT_TOKEN_", + ); + expect(run.result.stderr).not.toContain("ghp-host-secret-would-leak"); + expect(JSON.stringify(run.config)).not.toContain("ghp-host-secret-would-leak"); + }); +}); diff --git a/test/nemoclaw-start-guard-recovery.test.ts b/test/nemoclaw-start-guard-recovery.test.ts index cfdfbc43a17..b70caca08d4 100644 --- a/test/nemoclaw-start-guard-recovery.test.ts +++ b/test/nemoclaw-start-guard-recovery.test.ts @@ -175,6 +175,90 @@ describe("OpenClaw PID 1 guard-chain recovery", () => { } }); + // ── Recovery warning must reach the gateway log, not just stderr (#6065) ── + // + // #5874 moved recovery to a docker-IPC path where the warning was written to + // PID 1 stderr only; the live `issue-2478-crash-loop-recovery` E2E polls + // /tmp/gateway.log and went red. That target does not run on PR CI, so this + // mocked unit pins the file write (via the _NEMOCLAW_GATEWAY_LOG seam) in the + // PR gate to keep a refactor from silently regressing to stderr-only. + it("mirrors the guard-chain restore warning into the gateway log file", () => { + const source = fs.readFileSync(START_SCRIPT, "utf8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-guard-warn-")); + const gatewayLog = path.join(tmpDir, "gateway.log"); + try { + const script = [ + "set -uo pipefail", + `_NEMOCLAW_GATEWAY_LOG=${JSON.stringify(gatewayLog)}`, + // Force the chain-incomplete branch so the warning fires, and stub the + // downstream restore steps so this isolates the warning emission alone. + "openclaw_runtime_guard_chain_complete() { return 1; }", + "install_core_runtime_preloads() { return 0; }", + "write_messaging_runtime_setup_plan() { return 0; }", + "install_messaging_runtime_preloads() { return 0; }", + "verify_messaging_runtime_secret_scans() { return 0; }", + "write_runtime_shell_env() { return 0; }", + "validate_nemoclaw_tmp_permissions() { return 0; }", + extractShellFunction(source, "restore_openclaw_runtime_guard_chain"), + "rc=0; restore_openclaw_runtime_guard_chain || rc=$?", + 'printf "rc:%s\\n" "$rc"', + ].join("\n"); + + const result = spawnSync("bash", ["--noprofile", "--norc", "-c", script], { + encoding: "utf8", + timeout: 5000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("rc:0\n"); + // The marker must appear on stderr (operator console) AND in the gateway + // log file the recovery E2E observes. + expect(result.stderr).toContain("restoring library guards from packaged preloads"); + expect(fs.existsSync(gatewayLog)).toBe(true); + expect(fs.readFileSync(gatewayLog, "utf8")).toContain( + "[gateway-recovery] WARNING: /tmp guard chain missing or unsafe", + ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("does not emit the recovery warning when the guard chain is already complete", () => { + // Fence the branch: a healthy chain must stay silent so the log marker + // remains a true recovery signal rather than startup noise. + const source = fs.readFileSync(START_SCRIPT, "utf8"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-guard-quiet-")); + const gatewayLog = path.join(tmpDir, "gateway.log"); + try { + const script = [ + "set -uo pipefail", + `_NEMOCLAW_GATEWAY_LOG=${JSON.stringify(gatewayLog)}`, + "openclaw_runtime_guard_chain_complete() { return 0; }", + "install_core_runtime_preloads() { return 0; }", + "write_messaging_runtime_setup_plan() { return 0; }", + "install_messaging_runtime_preloads() { return 0; }", + "verify_messaging_runtime_secret_scans() { return 0; }", + "write_runtime_shell_env() { return 0; }", + "validate_nemoclaw_tmp_permissions() { return 0; }", + extractShellFunction(source, "restore_openclaw_runtime_guard_chain"), + "rc=0; restore_openclaw_runtime_guard_chain || rc=$?", + 'printf "rc:%s\\n" "$rc"', + ].join("\n"); + + const result = spawnSync("bash", ["--noprofile", "--norc", "-c", script], { + encoding: "utf8", + timeout: 5000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("rc:0\n"); + expect(result.stderr).not.toContain("restoring library guards"); + expect(fs.existsSync(gatewayLog)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + it("refuses an automatic respawn when guard restoration fails", () => { const source = fs.readFileSync(START_SCRIPT, "utf8"); const script = [ diff --git a/test/nemoclaw-start-reconcile.test.ts b/test/nemoclaw-start-reconcile.test.ts index 26137f43dd0..632b8853e08 100644 --- a/test/nemoclaw-start-reconcile.test.ts +++ b/test/nemoclaw-start-reconcile.test.ts @@ -321,6 +321,91 @@ describe("agent identity reconciliation with provider (#3175)", () => { expect(config.models.providers.inference.models[0].id).toBe("nvidia/new-model"); }); + // ── Explicit override wins over gateway reconciliation (#6065) ── + // + // #5874 re-architected gateway recovery and left reconcile running after + // apply_model_override with no guard, so its inference/-qualifying pass + // silently overwrote the user's explicit NEMOCLAW_MODEL_OVERRIDE. That + // regression only surfaced in the live `runtime-overrides` E2E, which does + // not run on PR CI. These mocked shell-units pin the guard in the PR gate. + + it("leaves an explicit NEMOCLAW_MODEL_OVERRIDE untouched even when the gateway reports a divergent model", () => { + const { result, config, hash } = runReconcile( + { + agents: { defaults: { model: { primary: "inference/user/explicit-choice" } } }, + models: { + providers: { + inference: { + api: "openai-completions", + models: [{ id: "user/explicit-choice", name: "inference/user/explicit-choice" }], + }, + }, + }, + }, + { + env: { NEMOCLAW_MODEL_OVERRIDE: "user/explicit-choice" }, + gatewayModel: "nvidia/nemotron-3-super-120b-a12b", + }, + ); + + expect(result.status).toBe(0); + // Without the guard, the gateway probe would rewrite primary AND models[0] + // to the divergent inference/-qualified value; the override must survive. + expect(config.agents.defaults.model.primary).toBe("inference/user/explicit-choice"); + expect(config.models.providers.inference.models[0].id).toBe("user/explicit-choice"); + expect(hash).toBe("oldhash\n"); + }); + + it("does not fall back to the in-file reconcile when NEMOCLAW_MODEL_OVERRIDE is set", () => { + // Even the legacy no-gateway path must be skipped: apply_model_override has + // already written the user's choice, so a stale file model must not win. + const { result, config, hash } = runReconcile( + { + agents: { defaults: { model: { primary: "inference/user/explicit-choice" } } }, + models: { + providers: { + inference: { + api: "openai-completions", + models: [ + { id: "nvidia/stale-file-model", name: "inference/nvidia/stale-file-model" }, + ], + }, + }, + }, + }, + { env: { NEMOCLAW_MODEL_OVERRIDE: "user/explicit-choice" } }, + ); + + expect(result.status).toBe(0); + expect(config.agents.defaults.model.primary).toBe("inference/user/explicit-choice"); + expect(hash).toBe("oldhash\n"); + }); + + it("still reconciles to the gateway model when NEMOCLAW_MODEL_OVERRIDE is unset", () => { + // Guard is scoped to explicit overrides only; the normal drift-correction + // path must keep working (regression fence around the early return itself). + const { result, config, hash } = runReconcile( + { + agents: { defaults: { model: { primary: "inference/nvidia-routed" } } }, + models: { + providers: { + inference: { + api: "openai-completions", + models: [{ id: "nvidia-routed", name: "inference/nvidia-routed" }], + }, + }, + }, + }, + { gatewayModel: "nvidia/nemotron-3-super-120b-a12b" }, + ); + + expect(result.status).toBe(0); + expect(config.agents.defaults.model.primary).toBe( + "inference/nvidia/nemotron-3-super-120b-a12b", + ); + expect(hash).not.toBe("oldhash\n"); + }); + it("falls back to the in-file reconcile when the gateway probe emits malformed JSON", () => { // A future packaging shift could ship an `openshell` shim that doesn't // implement `inference get --json` and returns junk on stdout. The diff --git a/test/no-unit-blocks-in-live-e2e.test.ts b/test/no-unit-blocks-in-live-e2e.test.ts new file mode 100644 index 00000000000..65dc92378e8 --- /dev/null +++ b/test/no-unit-blocks-in-live-e2e.test.ts @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { findLiveUnitBlocks, formatViolations } from "../scripts/checks/no-unit-blocks-in-live-e2e"; + +const FILE = "test/e2e/live/example.test.ts"; + +function linesFlagged(source: string): number[] { + return findLiveUnitBlocks(source, FILE).map((v) => v.line); +} + +describe("live E2E unit-block guard", () => { + it("flags the it(...) unit primitive parked in a live file", () => { + const source = [ + 'describe("local classifiers", () => {', + ' it("does something pure", () => {', + " expect(true).toBe(true);", + " });", + "});", + ].join("\n"); + expect(linesFlagged(source)).toEqual([2]); + }); + + it("flags it.each / it.only / it.skip member forms", () => { + const source = [ + 'it.each([1, 2])("case %s", () => {});', + 'it.only("focused", () => {});', + 'it.skip("skipped unit", () => {});', + ].join("\n"); + expect(linesFlagged(source)).toEqual([1, 2, 3]); + }); + + it("does not flag test(...) — the live-case primitive", () => { + const source = [ + 'test("live case", async ({ host }) => {});', + 'test("live case with module helpers", async () => {});', + 'test.skipIf(!shouldRunLiveE2E())("gated live case", async ({ sandbox }) => {});', + ].join("\n"); + expect(linesFlagged(source)).toEqual([]); + }); + + it("does not flag gated wrappers or the shouldRunLiveE2E ternary", () => { + const source = [ + "const liveTest = shouldRunLiveE2E() ? test : test.skip;", + 'liveTest("a gated live case", async ({ host }) => {});', + 'openClawTest("openclaw live case", async ({ sandbox }) => {});', + 'describe.sequential("live targets", () => {', + ' hermesTest("hermes live case", async ({ host }) => {});', + "});", + ].join("\n"); + expect(linesFlagged(source)).toEqual([]); + }); + + it("does not flag the vitest import or commented-out it(...) references", () => { + const source = [ + 'import { describe, it, test } from "vitest";', + '// it("a commented unit case", () => {});', + ' * it("a jsdoc example", () => {});', + ].join("\n"); + expect(linesFlagged(source)).toEqual([]); + }); + + it("does not match it inside a longer identifier", () => { + const source = [ + 'const wait = () => {}; wait("not a test");', + 'commitEditor("noop", () => {});', + ].join("\n"); + expect(linesFlagged(source)).toEqual([]); + }); + + it("formats a violation with file, line, and the offending text", () => { + const violations = findLiveUnitBlocks(' it("x", () => {});', FILE); + const rendered = formatViolations(violations); + expect(rendered).toContain(`${FILE}:1`); + expect(rendered).toContain('it("x"'); + expect(rendered).toContain("never runs"); + }); +}); diff --git a/test/ollama-auth-proxy-handler-helpers.ts b/test/ollama-auth-proxy-handler-helpers.ts new file mode 100644 index 00000000000..6a57e0d23bc --- /dev/null +++ b/test/ollama-auth-proxy-handler-helpers.ts @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Test harness helpers for ollama-auth-proxy-handler.test.ts. The stub backend, +// free-port probe, child-process proxy launcher/terminator, and the loopback +// request driver all branch, so they live here to keep the test body linear. + +import { type ChildProcess, spawn } from "node:child_process"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import path from "node:path"; + +export const PROXY_SCRIPT = path.resolve( + import.meta.dirname, + "..", + "scripts", + "ollama-auth-proxy.js", +); + +export interface BackendCapture { + method: string; + url: string; + headers: http.IncomingHttpHeaders; +} + +/** Start a loopback stub backend that records the request it received. */ +export function startBackend(): Promise<{ + server: http.Server; + port: number; + captured: BackendCapture[]; +}> { + const captured: BackendCapture[] = []; + const server = http.createServer((req, res) => { + captured.push({ + method: req.method ?? "", + url: req.url ?? "", + headers: { ...req.headers }, + }); + // Drain the body so piped client requests complete cleanly. + req.resume(); + req.on("end", () => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true, models: [] })); + }); + }); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + resolve({ server, port: (server.address() as AddressInfo).port, captured }); + }); + }); +} + +/** Grab an ephemeral free TCP port, then release it for the proxy to bind. */ +export function freePort(): Promise { + return new Promise((resolve, reject) => { + const probe = http.createServer(); + probe.once("error", reject); + probe.listen(0, "127.0.0.1", () => { + const port = (probe.address() as AddressInfo).port; + probe.close(() => resolve(port)); + }); + }); +} + +/** Spawn the real proxy script and wait until its listener accepts a connection. */ +export async function startProxy( + proxyPort: number, + backendPort: number, + token: string, +): Promise { + const child = spawn(process.execPath, [PROXY_SCRIPT], { + env: { + ...process.env, + OLLAMA_PROXY_TOKEN: token, + OLLAMA_PROXY_PORT: String(proxyPort), + OLLAMA_BACKEND_PORT: String(backendPort), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + await new Promise((resolve, reject) => { + let settled = false; + const timer = setTimeout(() => { + settled = true; + reject(new Error("proxy did not start in time")); + }, 5_000); + const tryConnect = (): void => { + if (settled) return; + const req = http.request( + { host: "127.0.0.1", port: proxyPort, path: "/", method: "GET" }, + (res) => { + res.resume(); + settled = true; + clearTimeout(timer); + resolve(); + }, + ); + req.on("error", () => { + if (!settled) setTimeout(tryConnect, 100); + }); + req.end(); + }; + child.once("exit", (code) => { + settled = true; + clearTimeout(timer); + reject(new Error(`proxy exited early with code ${code}`)); + }); + tryConnect(); + }); + return child; +} + +export async function terminate(child: ChildProcess | undefined): Promise { + if (!child || child.killed || child.exitCode !== null) return; + child.kill("SIGTERM"); + await new Promise((resolve) => { + const timer = setTimeout(() => { + if (!child.killed && child.exitCode === null) child.kill("SIGKILL"); + resolve(); + }, 2_000); + child.once("exit", () => { + clearTimeout(timer); + resolve(); + }); + }); +} + +export interface ProxyResponse { + status: number; + body: string; +} + +/** Issue a real request through the proxy on loopback. */ +export function request( + proxyPort: number, + options: { method?: string; path?: string; auth?: string; body?: string }, +): Promise { + return new Promise((resolve, reject) => { + const headers: Record = { host: "example.invalid" }; + if (options.auth !== undefined) headers.authorization = options.auth; + if (options.body !== undefined) headers["content-type"] = "application/json"; + const req = http.request( + { + host: "127.0.0.1", + port: proxyPort, + path: options.path ?? "/api/tags", + method: options.method ?? "GET", + headers, + }, + (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { + body += chunk; + }); + res.on("end", () => resolve({ status: res.statusCode ?? 0, body })); + }, + ); + req.on("error", reject); + if (options.body !== undefined) req.write(options.body); + req.end(); + }); +} diff --git a/test/ollama-auth-proxy-handler.test.ts b/test/ollama-auth-proxy-handler.test.ts new file mode 100644 index 00000000000..e221c457388 --- /dev/null +++ b/test/ollama-auth-proxy-handler.test.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Mocked unit coverage for the Bearer-token enforcement and header-stripping +// contract of scripts/ollama-auth-proxy.js. The live E2E target +// (test/e2e/live/ollama-auth-proxy.test.ts) exercises the same boundary but +// needs a real Ollama install plus a model pull; this pins the security- +// critical request-handler behavior hermetically. +// +// The proxy script is a standalone IIFE that binds a listener at load, so it +// cannot be required as a handler. Instead we spawn it as a real child process +// (unmodified production code) on an ephemeral port, point it at a tiny +// in-process stub HTTP backend, and drive real requests through it. No network +// beyond loopback; both servers and the child are torn down in afterEach. + +import type { ChildProcess } from "node:child_process"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + freePort, + request, + startBackend, + startProxy, + terminate, +} from "./ollama-auth-proxy-handler-helpers.ts"; + +const TOKEN = "unit-test-secret-token"; + +describe("ollama-auth-proxy request handler", () => { + let backend: Awaited> | undefined; + let proxy: ChildProcess | undefined; + let proxyPort = 0; + + beforeEach(async () => { + backend = await startBackend(); + proxyPort = await freePort(); + proxy = await startProxy(proxyPort, backend.port, TOKEN); + }); + + afterEach(async () => { + await terminate(proxy); + proxy = undefined; + await new Promise((resolve) => backend?.server.close(() => resolve())); + backend = undefined; + }); + + it("returns 401 when the Authorization header is missing", async () => { + const res = await request(proxyPort, { path: "/api/generate", method: "POST", body: "{}" }); + expect(res.status).toBe(401); + expect(backend?.captured).toHaveLength(0); + }); + + it("returns 401 when the Bearer token is wrong", async () => { + const res = await request(proxyPort, { path: "/api/generate", auth: "Bearer wrong-token" }); + expect(res.status).toBe(401); + expect(backend?.captured).toHaveLength(0); + }); + + it("returns 401 for unauthenticated /api/tags — no health-check bypass (#3338)", async () => { + const res = await request(proxyPort, { path: "/api/tags" }); + expect(res.status).toBe(401); + expect(backend?.captured).toHaveLength(0); + }); + + it("forwards to the backend on a correct Bearer token and strips authorization + host headers", async () => { + const res = await request(proxyPort, { + path: "/v1/chat/completions", + method: "POST", + auth: `Bearer ${TOKEN}`, + body: JSON.stringify({ model: "m", messages: [] }), + }); + expect(res.status).toBe(200); + expect(backend?.captured).toHaveLength(1); + const forwarded = backend?.captured[0]; + expect(forwarded?.method).toBe("POST"); + expect(forwarded?.url).toBe("/v1/chat/completions"); + // The auth header must never reach Ollama, and the client Host + // (example.invalid) must be dropped so it does not override the backend. + expect(forwarded?.headers.authorization).toBeUndefined(); + expect(forwarded?.headers.host).not.toBe("example.invalid"); + }); + + it("returns 401 without crashing on a non-ASCII auth header of equal length but different byte length (#4820)", async () => { + // "Bearer " + a multi-byte character string whose JS .length equals the + // expected string's .length but whose UTF-8 byte length differs. A naive + // string/length gate that fed unequal-length buffers to timingSafeEqual + // would throw and crash the 0.0.0.0-bound proxy. + const expected = `Bearer ${TOKEN}`; + const prefix = "Bearer "; + const restLen = expected.length - prefix.length; + const multiByte = prefix + "é".repeat(restLen); + expect(multiByte.length).toBe(expected.length); + expect(Buffer.byteLength(multiByte)).not.toBe(Buffer.byteLength(expected)); + + const res = await request(proxyPort, { path: "/api/tags", auth: multiByte }); + expect(res.status).toBe(401); + expect(backend?.captured).toHaveLength(0); + + // The proxy must still be alive and serve a subsequent valid request. + const ok = await request(proxyPort, { path: "/api/tags", auth: `Bearer ${TOKEN}` }); + expect(ok.status).toBe(200); + expect(proxy?.exitCode).toBeNull(); + }); + + it("returns 502 when the backend connection fails", async () => { + // Kill the backend so the forward connection is refused; a valid token + // then reaches the backend request that errors → 502. + await new Promise((resolve) => backend?.server.close(() => resolve())); + const res = await request(proxyPort, { path: "/api/tags", auth: `Bearer ${TOKEN}` }); + expect(res.status).toBe(502); + expect(res.body).toMatch(/Ollama backend error/); + expect(proxy?.exitCode).toBeNull(); + }); +}); diff --git a/test/ollama-proxy-recovery.test.ts b/test/ollama-proxy-recovery.test.ts index 76ef41e4bb5..da9621496fe 100644 --- a/test/ollama-proxy-recovery.test.ts +++ b/test/ollama-proxy-recovery.test.ts @@ -383,4 +383,247 @@ console.log(JSON.stringify({ assert.equal(payload.proxySpawns[0].env.OLLAMA_PROXY_PORT, "11435"); assert.equal(payload.proxySpawns[0].env.OLLAMA_BACKEND_PORT, "11434"); }); + + it("persists the proxy token at mode 0600 matching the running token (#2553)", () => { + // startOllamaAuthProxy() mints an in-memory token; persistProxyToken() is + // the seam that writes it to disk. Assert the on-disk file (a) exists at + // mode 0600 and (b) matches the token the runner reports as current — the + // token-file invariant otherwise only exercised by the live E2E (phase 7). + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ollama-proxy-persist-")); + const scriptPath = path.join(tmpDir, "persist-token-check.js"); + const proxyPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "inference", "ollama", "proxy.ts"), + ); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + + const script = String.raw` +const fs = require("node:fs"); +const path = require("node:path"); +const childProcess = require("child_process"); +const runner = require(${runnerPath}); + +childProcess.spawn = () => ({ pid: 7777, unref() {} }); +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : command; + if (text.includes("lsof") && text.includes("11435")) return ""; + if (text.includes("ps -p 7777")) return "node /repo/scripts/ollama-auth-proxy.js"; + return ""; +}; +runner.run = () => ({ status: 0, stdout: "", stderr: "" }); + +const origSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (...args) => { + if (args[0] === "sleep") return { status: 0, stdout: "", stderr: "" }; + if (args[0] === "nc") return { error: null, status: 0, stdout: "", stderr: "" }; + if (args[0] === "curl") { + const argv = Array.isArray(args[1]) ? args[1] : []; + // authed probe → 200 (accepted); unauth probe → 401 (rejected). + return { status: 0, stdout: argv.includes("--config") ? "200" : "401", stderr: "" }; + } + return origSpawnSync(...args); +}; + +const proxy = require(${proxyPath}); +const started = proxy.startOllamaAuthProxy(); +// startOllamaAuthProxy intentionally holds the token in memory only; the +// onboarding flow persists it once the provider is confirmed. Exercise that seam. +const running = proxy.getOllamaProxyToken(); +proxy.persistProxyToken(running); + +const tokenPath = path.join(process.env.HOME, ".nemoclaw", "ollama-proxy-token"); +const stat = fs.statSync(tokenPath); +console.log(JSON.stringify({ + started, + mode: (stat.mode & 0o777).toString(8), + fileToken: fs.readFileSync(tokenPath, "utf8").trim(), + runningToken: running, +})); +`; + fs.writeFileSync(scriptPath, script); + + const childEnv: NodeJS.ProcessEnv = { ...process.env, HOME: tmpDir }; + delete childEnv.NEMOCLAW_OLLAMA_PROXY_PORT; + delete childEnv.NEMOCLAW_OLLAMA_PORT; + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: childEnv, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson<{ + started: boolean; + mode: string; + fileToken: string; + runningToken: string; + }>(result.stdout); + assert.equal(payload.started, true); + // Token file is 0600 and its contents match the running token. + assert.equal(payload.mode, "600"); + assert.ok(payload.fileToken.length > 0, "expected a non-empty persisted token"); + assert.equal(payload.fileToken, payload.runningToken); + }); + + it("restart preserves a 0600 token file whose contents match the respawned token (#2553)", () => { + // A stale recorded pid forces a restart. Beyond spawning with the persisted + // token (covered above), assert the lifecycle invariant: the token file + // survives the restart at mode 0600 and the respawned proxy is launched with + // exactly that file token — the persisted token round-trips into the child. + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ollama-proxy-restart-mode-")); + const scriptPath = path.join(tmpDir, "restart-mode-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + + const script = String.raw` +const fs = require("node:fs"); +const path = require("node:path"); +const childProcess = require("child_process"); +const runner = require(${runnerPath}); + +let spawnedToken = null; +childProcess.spawn = (cmd, args, opts = {}) => { + spawnedToken = opts.env && opts.env.OLLAMA_PROXY_TOKEN; + return { pid: 4242, unref() {} }; +}; +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : command; + if (text.includes("ps -p 99999")) return ""; + if (text.includes("ps -p 4242")) return "node /tmp/ollama-auth-proxy.js"; + if (text.includes("lsof -ti :11435")) return ""; + return ""; +}; +runner.run = () => ({ status: 0, stdout: "", stderr: "" }); + +const origSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (...args) => { + if (args[0] === "curl") return { status: 0, stdout: "200", stderr: "" }; + if (args[0] === "sleep") return { status: 0, stdout: "", stderr: "" }; + return origSpawnSync(...args); +}; + +const stateDir = path.join(process.env.HOME, ".nemoclaw"); +fs.mkdirSync(stateDir, { recursive: true }); +const tokenPath = path.join(stateDir, "ollama-proxy-token"); +fs.writeFileSync(tokenPath, "persisted-token\n", { mode: 0o600 }); +fs.writeFileSync(path.join(stateDir, "ollama-auth-proxy.pid"), "99999\n", { mode: 0o600 }); + +const onboard = require(${onboardPath}); +onboard.ensureOllamaAuthProxy(); + +const stat = fs.statSync(tokenPath); +console.log(JSON.stringify({ + spawnedToken, + mode: (stat.mode & 0o777).toString(8), + fileToken: fs.readFileSync(tokenPath, "utf8").trim(), +})); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { ...process.env, HOME: tmpDir }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson<{ spawnedToken: string; mode: string; fileToken: string }>( + result.stdout, + ); + // Restart reuses the persisted token; the file is untouched at 0600. + assert.equal(payload.mode, "600"); + assert.equal(payload.fileToken, "persisted-token"); + assert.equal(payload.spawnedToken, "persisted-token"); + }); + + it("repairs a divergent on-disk token by restarting with the file token (#2553)", () => { + // Divergence: the running proxy holds a token that no longer matches the + // authoritative on-disk token (e.g. after a failed re-onboard rewrote the + // file). The file token probe returns 401, so ensureOllamaAuthProxy detects + // the divergence, reclaims the stale proxy, and restarts it with the FILE + // token — the on-disk value is authoritative, not whatever was running. + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ollama-proxy-divergent-")); + const scriptPath = path.join(tmpDir, "divergent-token-check.js"); + const proxyPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "inference", "ollama", "proxy.ts"), + ); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + + const script = String.raw` +const fs = require("node:fs"); +const path = require("node:path"); +const childProcess = require("child_process"); +const runner = require(${runnerPath}); + +let spawnedToken = null; +const runCommands = []; +childProcess.spawn = (cmd, args, opts = {}) => { + spawnedToken = opts.env && opts.env.OLLAMA_PROXY_TOKEN; + return { pid: 5000, unref() {} }; +}; +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : command; + if (text.includes("ps -p 4242")) return "node /tmp/ollama-auth-proxy.js"; + if (text.includes("ps -p 5000")) return "node /tmp/ollama-auth-proxy.js"; + if (text.includes("lsof -ti :11435")) return ""; + return ""; +}; +runner.run = (command) => { runCommands.push(command); return { status: 0, stdout: "", stderr: "" }; }; + +let curlCalls = 0; +const origSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (...args) => { + if (args[0] === "curl") { + curlCalls += 1; + // The running proxy holds a DIFFERENT token: first probe (file token) → 401 + // (divergence), post-restart probe → 200 (repaired). + return { status: 0, stdout: curlCalls === 1 ? "401" : "200", stderr: "" }; + } + if (args[0] === "sleep") return { status: 0, stdout: "", stderr: "" }; + return origSpawnSync(...args); +}; + +const stateDir = path.join(process.env.HOME, ".nemoclaw"); +fs.mkdirSync(stateDir, { recursive: true }); +const tokenPath = path.join(stateDir, "ollama-proxy-token"); +// The authoritative on-disk token, divergent from whatever ran before. +fs.writeFileSync(tokenPath, "new-file-token\n", { mode: 0o600 }); +fs.writeFileSync(path.join(stateDir, "ollama-auth-proxy.pid"), "4242\n", { mode: 0o600 }); + +const proxy = require(${proxyPath}); +proxy.ensureOllamaAuthProxy(); + +const stat = fs.statSync(tokenPath); +console.log(JSON.stringify({ + spawnedToken, + runCommands, + mode: (stat.mode & 0o777).toString(8), + fileToken: fs.readFileSync(tokenPath, "utf8").trim(), +})); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { ...process.env, HOME: tmpDir }, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = parseStdoutJson<{ + spawnedToken: string; + runCommands: string[][]; + mode: string; + fileToken: string; + }>(result.stdout); + // The stale proxy is reclaimed and the repair restart uses the FILE token. + assert.deepEqual(payload.runCommands[0], ["kill", "4242"]); + assert.equal(payload.spawnedToken, "new-file-token"); + // The authoritative token file is preserved at 0600. + assert.equal(payload.mode, "600"); + assert.equal(payload.fileToken, "new-file-token"); + }); }); diff --git a/test/openclaw-device-approval-policy.test.ts b/test/openclaw-device-approval-policy.test.ts index 975befc53cc..7e72ff323e7 100644 --- a/test/openclaw-device-approval-policy.test.ts +++ b/test/openclaw-device-approval-policy.test.ts @@ -9,6 +9,12 @@ import { describe, expect, it } from "vitest"; const REPO_ROOT = path.resolve(import.meta.dirname, ".."); const POLICY_PATH = path.join(REPO_ROOT, "scripts", "lib", "openclaw_device_approval_policy.py"); +function hasPython3(): boolean { + return spawnSync("sh", ["-c", "command -v python3"], { stdio: "ignore" }).status === 0; +} + +const HAS_PYTHON3 = hasPython3(); + function evaluatePolicy(devices: unknown[], env: Record = {}) { const script = ` import importlib.util @@ -41,8 +47,57 @@ print(json.dumps(payload, default=lambda value: sorted(value))) return JSON.parse(result.stdout); } +function callDecision(device: unknown) { + const script = ` +import importlib.util +import json +import sys + +policy_path = sys.argv[1] +device = json.loads(sys.argv[2]) +spec = importlib.util.spec_from_file_location("openclaw_device_approval_policy", policy_path) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +result = module.approval_request_decision(device) +result["scopes"] = sorted(result["scopes"]) +print(json.dumps(result, sort_keys=True)) +`; + return spawnSync("python3", ["-", POLICY_PATH, JSON.stringify(device)], { + encoding: "utf-8", + input: script, + timeout: 10_000, + }); +} + +function callGatewayEnv(sourceEnv: Record) { + const script = ` +import importlib.util +import json +import sys + +policy_path = sys.argv[1] +source_env = json.loads(sys.argv[2]) +spec = importlib.util.spec_from_file_location("openclaw_device_approval_policy", policy_path) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +result = module.gateway_approval_env(source_env) +print(json.dumps(result, sort_keys=True)) +`; + return spawnSync("python3", ["-", POLICY_PATH, JSON.stringify(sourceEnv)], { + encoding: "utf-8", + input: script, + timeout: 10_000, + }); +} + +function decisionOf(device: unknown) { + const proc = callDecision(device); + expect(proc.status).toBe(0); + return JSON.parse(proc.stdout); +} + describe("OpenClaw device approval policy", () => { - it("keeps allowlisting and gateway-environment stripping pure", () => { + it.skipIf(!HAS_PYTHON3)("keeps allowlisting and gateway-environment stripping pure", () => { const payload = evaluatePolicy([ { requestId: "bounded-cli", @@ -94,3 +149,107 @@ describe("OpenClaw device approval policy", () => { expect(payload.has_recovery).toBe(false); }); }); + +describe("approval_request_decision scope-upgrade gate (#4462)", () => { + it.skipIf(!HAS_PYTHON3)("allows a known client requesting the exact operator allowlist", () => { + const decision = decisionOf({ + clientId: "openclaw-control-ui", + clientMode: "webchat", + scopes: ["operator.pairing", "operator.read", "operator.write"], + }); + expect(decision.allowed).toBe(true); + expect(decision.reason).toBe("allowlisted"); + expect(decision.scopes).toEqual(["operator.pairing", "operator.read", "operator.write"]); + }); + + it.skipIf(!HAS_PYTHON3)("rejects an unknown client regardless of the claimed mode", () => { + const decision = decisionOf({ + clientId: "rogue-client", + clientMode: "cli", + scopes: ["operator.read"], + }); + expect(decision.allowed).toBe(false); + expect(decision.reason).toBe("unknown-client"); + expect(decision.scopes).toEqual([]); + }); + + it.skipIf(!HAS_PYTHON3)("rejects a scope superset that exceeds the allowlist", () => { + const decision = decisionOf({ + clientId: "openclaw-control-ui", + clientMode: "webchat", + scopes: ["operator.pairing", "operator.read", "operator.write", "operator.delete"], + }); + expect(decision.allowed).toBe(false); + expect(decision.reason).toBe("disallowed-scopes"); + }); + + it.skipIf(!HAS_PYTHON3)("allows a scope subset of the allowlist", () => { + const decision = decisionOf({ + clientId: "openclaw-control-ui", + clientMode: "webchat", + scopes: ["operator.read"], + }); + expect(decision.allowed).toBe(true); + expect(decision.reason).toBe("allowlisted"); + expect(decision.scopes).toEqual(["operator.read"]); + }); + + it.skipIf(!HAS_PYTHON3)("rejects malformed non-list scopes", () => { + const decision = decisionOf({ + clientId: "openclaw-control-ui", + clientMode: "webchat", + scopes: "operator.read", + }); + expect(decision.allowed).toBe(false); + expect(decision.reason).toBe("malformed-scopes"); + }); + + it.skipIf(!HAS_PYTHON3)("rejects any operator.admin escalation from a known client", () => { + const decision = decisionOf({ + clientId: "openclaw-control-ui", + clientMode: "webchat", + scopes: ["operator.pairing", "operator.read", "operator.write", "operator.admin"], + }); + expect(decision.allowed).toBe(false); + expect(decision.reason).toBe("disallowed-scopes"); + }); + + it.skipIf(!HAS_PYTHON3)("rejects an operator.admin-only request from a known client", () => { + const decision = decisionOf({ + clientId: "openclaw-control-ui", + clientMode: "webchat", + scopes: ["operator.admin"], + }); + expect(decision.allowed).toBe(false); + expect(decision.reason).toBe("disallowed-scopes"); + }); +}); + +describe("gateway_approval_env sanitization (#4462)", () => { + it.skipIf(!HAS_PYTHON3)("strips the three gateway keys and preserves everything else", () => { + const proc = callGatewayEnv({ + OPENCLAW_GATEWAY_URL: "http://gateway:8080", + OPENCLAW_GATEWAY_PORT: "8080", + OPENCLAW_GATEWAY_TOKEN: "secret-token", + PATH: "/usr/bin", + OPENCLAW_STATE_DIR: "/sandbox/.openclaw", + HOME: "/home/agent", + }); + expect(proc.status).toBe(0); + const env = JSON.parse(proc.stdout); + expect(env).not.toHaveProperty("OPENCLAW_GATEWAY_URL"); + expect(env).not.toHaveProperty("OPENCLAW_GATEWAY_PORT"); + expect(env).not.toHaveProperty("OPENCLAW_GATEWAY_TOKEN"); + expect(env).toEqual({ + PATH: "/usr/bin", + OPENCLAW_STATE_DIR: "/sandbox/.openclaw", + HOME: "/home/agent", + }); + }); + + it.skipIf(!HAS_PYTHON3)("is a no-op when no gateway keys are present", () => { + const proc = callGatewayEnv({ PATH: "/usr/bin", HOME: "/home/agent" }); + expect(proc.status).toBe(0); + expect(JSON.parse(proc.stdout)).toEqual({ PATH: "/usr/bin", HOME: "/home/agent" }); + }); +}); diff --git a/test/runtime-shell.test.ts b/test/runtime-shell.test.ts index 86bdaa0ef3b..3ae84432779 100644 --- a/test/runtime-shell.test.ts +++ b/test/runtime-shell.test.ts @@ -1,11 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, it, expect } from "vitest"; +import { type SpawnSyncReturns, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { spawnSync, type SpawnSyncReturns } from "node:child_process"; +import { describe, expect, it } from "vitest"; const RUNTIME_SH = path.join(import.meta.dirname, "..", "scripts", "lib", "runtime.sh"); @@ -162,6 +162,44 @@ describe("shell runtime helpers", () => { expect(result.status).not.toBe(0); }); + // An out-of-range or non-numeric NEMOCLAW_VLLM_PORT / NEMOCLAW_OLLAMA_PORT + // must be rejected by _validate_port so get_local_provider_base_url and + // check_local_provider_health fail closed instead of building a bogus URL. + it.each([ + { name: "NEMOCLAW_VLLM_PORT", value: "99999" }, + { name: "NEMOCLAW_VLLM_PORT", value: "0" }, + { name: "NEMOCLAW_VLLM_PORT", value: "abc" }, + { name: "NEMOCLAW_OLLAMA_PORT", value: "99999" }, + { name: "NEMOCLAW_OLLAMA_PORT", value: "0" }, + { name: "NEMOCLAW_OLLAMA_PORT", value: "abc" }, + ])("get_local_provider_base_url fails closed on invalid $name=$value", ({ name, value }) => { + const provider = name === "NEMOCLAW_VLLM_PORT" ? "vllm-local" : "ollama-local"; + const result = runShell(`source "${RUNTIME_SH}"; get_local_provider_base_url ${provider}`, { + [name]: value, + }); + + expect(result.status).not.toBe(0); + expect(result.stdout.trim()).toBe(""); + expect(result.stderr).toContain(`Invalid ${name}=${value} (expected 1024-65535)`); + }); + + it.each([ + { name: "NEMOCLAW_VLLM_PORT", value: "99999" }, + { name: "NEMOCLAW_VLLM_PORT", value: "0" }, + { name: "NEMOCLAW_VLLM_PORT", value: "abc" }, + { name: "NEMOCLAW_OLLAMA_PORT", value: "99999" }, + { name: "NEMOCLAW_OLLAMA_PORT", value: "0" }, + { name: "NEMOCLAW_OLLAMA_PORT", value: "abc" }, + ])("check_local_provider_health fails closed on invalid $name=$value", ({ name, value }) => { + const provider = name === "NEMOCLAW_VLLM_PORT" ? "vllm-local" : "ollama-local"; + const result = runShell(`source "${RUNTIME_SH}"; check_local_provider_health ${provider}`, { + [name]: value, + }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain(`Invalid ${name}=${value} (expected 1024-65535)`); + }); + it("returns the first non-loopback nameserver", () => { const result = runShell( `source "${RUNTIME_SH}"; first_non_loopback_nameserver $'nameserver 127.0.0.11\\nnameserver 10.0.0.2'`, diff --git a/tsconfig.runtime-preloads.json b/tsconfig.runtime-preloads.json index 86fa7315889..e2c55ba7540 100644 --- a/tsconfig.runtime-preloads.json +++ b/tsconfig.runtime-preloads.json @@ -16,5 +16,8 @@ "noEmitOnError": true }, "include": ["src/lib/messaging/channels/*/runtime/*.ts"], - "exclude": [] + "exclude": [ + "src/lib/messaging/channels/*/runtime/*.test.ts", + "src/lib/messaging/channels/*/runtime/*-test-helpers.ts" + ] }