diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 0a034a98a64..5d9efa05c0d 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -565,7 +565,12 @@ $$nemoclaw dcode-sandbox agent -n "Summarize this repository" ``` The wrapper inherits the remote command's exit code, so host-side pipelines can branch on it. -Streaming forwards whatever the in-sandbox agent command emits on `stdout`; the wrapper adds no buffering. +For normal turns, streaming forwards whatever the in-sandbox agent command emits on `stdout`; the wrapper adds no buffering. +When the top-level OpenClaw `--json` output flag is present, the wrapper uses a captured no-TTY path with a `64 MiB` buffer so `stdout` stays parseable JSON. +Raw `stderr` is forwarded, and failed-tool or untrusted-child provenance found in the stdout JSON is appended to `stderr`. +Literal `--json` values consumed by flags such as `-m` or `--reply-channel`, or arguments after `--`, stay on the normal passthrough path. +Documented value flags written as `--flag=value`, such as `--session-id=s1`, are recognized the same way as separated value flags. +If an unrecognized OpenClaw option appears before `--json`, NemoClaw also keeps the command on the normal passthrough path so OpenClaw remains the argv source of truth. Common OpenClaw flags include `-m `, `--session-id `, `--agent `, `--model `, `--thinking `, `--json`, `--deliver`, `--reply-channel `, and `--timeout `. For OpenClaw sandboxes and registry fallbacks, `$$nemoclaw agent --help` prints the wrapper-level summary locally. diff --git a/src/commands/sandbox/agent.ts b/src/commands/sandbox/agent.ts index 09ae40d296d..0324064d36a 100644 --- a/src/commands/sandbox/agent.ts +++ b/src/commands/sandbox/agent.ts @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { printAgentPassthroughHelp } from "../../lib/actions/sandbox/agent/passthrough-help"; import { runAgentPassthrough } from "../../lib/actions/sandbox/agent/passthrough"; +import { printAgentPassthroughHelp } from "../../lib/actions/sandbox/agent/passthrough-help"; import { NemoClawCommand } from "../../lib/cli/nemoclaw-oclif-command"; export default class SandboxAgentCommand extends NemoClawCommand { @@ -10,7 +10,7 @@ export default class SandboxAgentCommand extends NemoClawCommand { static strict = false; static summary = "Run one agent turn non-interactively in a sandbox"; static description = - "Pass through to the sandbox's registered agent command via `openshell sandbox exec`. OpenClaw sandboxes run `openclaw agent`; terminal-runtime sandboxes run their manifest-declared interactive command, such as `dcode` for LangChain Deep Agents Code. Stream the agent's response back to stdout without owning a TTY; useful for driving the sandbox from another process (CI job, multi-agent platform, evaluation harness). Hermes sandboxes exit non-zero with a redirect to the OpenAI-compatible API on port 8642 inside the sandbox."; + "Pass through to the sandbox's registered agent command via `openshell sandbox exec`. OpenClaw sandboxes run `openclaw agent`; terminal-runtime sandboxes run their manifest-declared interactive command, such as `dcode` for LangChain Deep Agents Code. Normal turns stream the agent's response without owning a TTY; top-level OpenClaw `--json` uses a captured path that preserves JSON stdout and appends provenance to stderr. Useful for driving the sandbox from another process (CI job, multi-agent platform, evaluation harness). Hermes sandboxes exit non-zero with a redirect to the OpenAI-compatible API on port 8642 inside the sandbox."; static usage = [" [agent-flags...]"]; static examples = [ '<%= config.bin %> sandbox agent alpha --agent work -m "Summarise README.md"', diff --git a/src/lib/actions/sandbox/agent/passthrough-json.test.ts b/src/lib/actions/sandbox/agent/passthrough-json.test.ts new file mode 100644 index 00000000000..be9e4931d40 --- /dev/null +++ b/src/lib/actions/sandbox/agent/passthrough-json.test.ts @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { runAgentJsonPassthrough } from "./passthrough-json"; + +describe("runAgentJsonPassthrough", () => { + function makeProc() { + const stdout: string[] = []; + const stderr: string[] = []; + const exit = vi.fn((code: number) => { + throw new Error(`__exit:${code}`); + }); + return { + exit, + proc: { + exit: exit as unknown as (code: number) => never, + stdout: { write: (value: string) => stdout.push(value) }, + stderr: { write: (value: string) => stderr.push(value) }, + }, + stderr, + stdout, + }; + } + + it("preserves OpenClaw JSON stdout and appends failed-tool provenance to stderr", () => { + const payload = JSON.stringify({ + result: { + messages: [ + { + role: "toolResult", + type: "toolResult", + toolName: "exec", + toolCallId: "call_missing", + isError: true, + text: "exec failed: node-not-real: not found", + }, + ], + payloads: [{ text: "Saved successfully." }], + }, + }); + const spawnSync = vi.fn(() => ({ + status: 0, + signal: null, + stdout: payload, + stderr: "openclaw warning\n", + pid: 123, + output: [null, payload, "openclaw warning\n"], + })); + const { exit, proc, stderr, stdout } = makeProc(); + + expect(() => + runAgentJsonPassthrough("alpha", ["openclaw", "agent", "--json"], proc, { + getOpenshellBinary: () => "/usr/local/bin/openshell", + spawnSync, + }), + ).toThrow("__exit:0"); + + expect(spawnSync).toHaveBeenCalledWith( + "/usr/local/bin/openshell", + ["sandbox", "exec", "--name", "alpha", "--no-tty", "--", "openclaw", "agent", "--json"], + expect.objectContaining({ + encoding: "utf-8", + maxBuffer: 64 * 1024 * 1024, + stdio: ["inherit", "pipe", "pipe"], + }), + ); + expect(stdout.join("")).toBe(payload); + expect(() => JSON.parse(stdout.join(""))).not.toThrow(); + expect(stderr.join("")).toContain("openclaw warning"); + expect(stderr.join("")).toContain("[openclaw provenance] failed tool result"); + expect(stderr.join("")).toContain("node-not-real"); + expect(exit).toHaveBeenCalledWith(0); + }); + + it("surfaces spawn errors and exits with the computed transport failure code", () => { + const spawnSync = vi.fn(() => ({ + status: null, + signal: null, + stdout: "", + stderr: "", + error: new Error("spawnSync openshell ENOENT"), + pid: 0, + output: [null, "", ""], + })); + const { exit, proc, stderr } = makeProc(); + + expect(() => + runAgentJsonPassthrough("alpha", ["openclaw", "agent", "--json"], proc, { + getOpenshellBinary: () => "openshell", + spawnSync, + }), + ).toThrow("__exit:1"); + + expect(stderr.join("")).toContain("Failed to invoke openshell"); + expect(stderr.join("")).toContain("spawnSync openshell ENOENT"); + expect(exit).toHaveBeenCalledWith(1); + }); + + it("does not treat stderr JSON diagnostics as agent provenance", () => { + const stdoutPayload = JSON.stringify({ result: { payloads: [{ text: "OK" }] } }); + const stderrPayload = JSON.stringify({ + messages: [ + { + role: "toolResult", + type: "toolResult", + toolName: "stderr-diagnostic", + toolCallId: "call_stderr", + isError: true, + text: "this was not part of stdout JSON", + }, + ], + }); + const spawnSync = vi.fn(() => ({ + status: 0, + signal: null, + stdout: stdoutPayload, + stderr: stderrPayload, + pid: 123, + output: [null, stdoutPayload, stderrPayload], + })); + const { proc, stderr } = makeProc(); + + expect(() => + runAgentJsonPassthrough("alpha", ["openclaw", "agent", "--json"], proc, { + getOpenshellBinary: () => "/usr/local/bin/openshell", + spawnSync, + }), + ).toThrow("__exit:0"); + + expect(stderr.join("")).toContain("stderr-diagnostic"); + expect(stderr.join("")).not.toContain("[openclaw provenance]"); + }); + + it("preserves forwarded output and remote exit code when provenance parsing fails", () => { + const stdoutPayload = JSON.stringify({ result: { payloads: [{ text: "OK" }] } }); + const spawnSync = vi.fn(() => ({ + status: 7, + signal: null, + stdout: stdoutPayload, + stderr: "openclaw warning", + pid: 123, + output: [null, stdoutPayload, "openclaw warning"], + })); + const { exit, proc, stderr, stdout } = makeProc(); + + expect(() => + runAgentJsonPassthrough("alpha", ["openclaw", "agent", "--json"], proc, { + getOpenshellBinary: () => "/usr/local/bin/openshell", + provenanceLines: () => { + throw new RangeError("Maximum call stack size exceeded"); + }, + spawnSync, + }), + ).toThrow("__exit:7"); + + expect(stdout.join("")).toBe(stdoutPayload); + expect(stderr.join("")).toContain("openclaw warning"); + expect(stderr.join("")).toContain( + "[openclaw provenance] skipped provenance extraction after parser failure.", + ); + expect(exit).toHaveBeenCalledWith(7); + }); +}); diff --git a/src/lib/actions/sandbox/agent/passthrough-json.ts b/src/lib/actions/sandbox/agent/passthrough-json.ts new file mode 100644 index 00000000000..63c4a17d1cb --- /dev/null +++ b/src/lib/actions/sandbox/agent/passthrough-json.ts @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync, type SpawnSyncOptions, type SpawnSyncReturns } from "node:child_process"; + +import { openClawAgentJsonProvenanceLines } from "../../../openclaw/agent-json-provenance"; +import { buildOpenshellExecArgs, computeExitCode } from "../exec"; + +const AGENT_JSON_MAX_BUFFER_BYTES = 64 * 1024 * 1024; + +export type AgentJsonPassthroughProcess = { + exit(code: number): never; + stdout: { write(s: string): unknown }; + stderr: { write(s: string): unknown }; +}; + +export type AgentJsonPassthroughDeps = { + getOpenshellBinary?: () => string; + provenanceLines?: (raw: string) => string[]; + spawnSync?: ( + command: string, + args: readonly string[], + options: SpawnSyncOptions, + ) => SpawnSyncReturns; +}; + +function text(value: string | Buffer | null | undefined): string { + if (Buffer.isBuffer(value)) return value.toString("utf-8"); + return typeof value === "string" ? value : ""; +} + +function defaultGetOpenshellBinary(): string { + // Lazy require keeps this module unit-testable under Vitest's TS loader; the + // OpenShell runtime imports runner/platform modules that only exist in built + // CLI layouts. + const runtime = + require("../../../adapters/openshell/runtime") as typeof import("../../../adapters/openshell/runtime"); + return runtime.getOpenshellBinary(); +} + +function writeProvenanceBlock( + proc: AgentJsonPassthroughProcess, + stderr: string, + lines: readonly string[], +): void { + if (lines.length === 0) return; + proc.stderr.write(`${stderr && !stderr.endsWith("\n") ? "\n" : ""}${lines.join("\n")}\n`); +} + +export function runAgentJsonPassthrough( + sandboxName: string, + command: readonly string[], + proc: AgentJsonPassthroughProcess = process, + deps: AgentJsonPassthroughDeps = {}, +): never { + const binary = (deps.getOpenshellBinary ?? defaultGetOpenshellBinary)(); + const spawnSyncImpl = deps.spawnSync ?? spawnSync; + const result = spawnSyncImpl( + binary, + buildOpenshellExecArgs(sandboxName, command, { tty: false }), + { + encoding: "utf-8", + maxBuffer: AGENT_JSON_MAX_BUFFER_BYTES, + stdio: ["inherit", "pipe", "pipe"], + }, + ); + const stdout = text(result.stdout); + const stderr = text(result.stderr); + if (stdout) proc.stdout.write(stdout); + if (stderr) proc.stderr.write(stderr); + + try { + writeProvenanceBlock( + proc, + stderr, + (deps.provenanceLines ?? openClawAgentJsonProvenanceLines)(stdout), + ); + } catch { + writeProvenanceBlock(proc, stderr, [ + "[openclaw provenance] skipped provenance extraction after parser failure.", + ]); + } + + const { code, errorMessage } = computeExitCode(result); + if (errorMessage) { + proc.stderr.write(` Failed to invoke openshell: ${errorMessage}\n`); + proc.stderr.write(" Ensure 'openshell' is installed and on PATH.\n"); + } + return proc.exit(code); +} diff --git a/src/lib/actions/sandbox/agent/passthrough.test.ts b/src/lib/actions/sandbox/agent/passthrough.test.ts index 7ea02e7f7a7..62e171cda1b 100644 --- a/src/lib/actions/sandbox/agent/passthrough.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough.test.ts @@ -33,7 +33,7 @@ vi.mock("../../../agent/defs", () => ({ loadAgent: loadAgentMock, })); -import { runAgentPassthrough } from "./passthrough"; +import { type AgentPassthroughDeps, runAgentPassthrough } from "./passthrough"; describe("runAgentPassthrough", () => { beforeEach(() => { @@ -71,12 +71,177 @@ describe("runAgentPassthrough", () => { it("forwards extraArgs verbatim to `openclaw agent` for OpenClaw sandboxes with --no-tty enforced", async () => { getSandboxMock.mockReturnValueOnce({ agent: "openclaw" }); await runAgentPassthrough("alpha", { - extraArgs: ["--agent", "work", "--session-id", "s-1", "-m", "ping", "--json"], + extraArgs: ["--agent", "work", "--session-id", "s-1", "-m", "ping"], }); expect(ensureLiveMock).toHaveBeenCalledWith("alpha", { allowNonReadyPhase: true }); expect(execMock).toHaveBeenCalledWith( + "alpha", + ["openclaw", "agent", "--agent", "work", "--session-id", "s-1", "-m", "ping"], + { tty: false }, + ); + }); + + it("uses the captured JSON path for `openclaw agent --json` so provenance can be emitted on stderr", async () => { + const execJson = vi.fn(() => { + throw new Error("__exit:0"); + }); + getSandboxMock.mockReturnValueOnce({ agent: "openclaw" }); + const { proc } = makeProcMock(); + + await expect( + runAgentPassthrough( + "alpha", + { + extraArgs: ["--agent", "work", "--session-id", "s-1", "-m", "ping", "--json"], + }, + { execJson, process: proc }, + ), + ).rejects.toThrow("__exit:0"); + + expect(ensureLiveMock).toHaveBeenCalledWith("alpha", { allowNonReadyPhase: true }); + expect(execMock).not.toHaveBeenCalled(); + expect(execJson).toHaveBeenCalledWith( "alpha", ["openclaw", "agent", "--agent", "work", "--session-id", "s-1", "-m", "ping", "--json"], + expect.objectContaining({ stderr: proc.stderr }), + ); + }); + + it("keeps --json as a message value on the normal passthrough path", async () => { + const execJson = vi.fn(((): never => { + throw new Error("__unexpected-json"); + }) as NonNullable); + getSandboxMock.mockReturnValueOnce({ agent: "openclaw" }); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "work", "-m", "--json"] }, + { execJson }, + ); + + expect(execJson).not.toHaveBeenCalled(); + expect(execMock).toHaveBeenCalledWith( + "alpha", + ["openclaw", "agent", "--agent", "work", "-m", "--json"], + { tty: false }, + ); + }); + + it("keeps --json after the argv terminator on the normal passthrough path", async () => { + const execJson = vi.fn(((): never => { + throw new Error("__unexpected-json"); + }) as NonNullable); + getSandboxMock.mockReturnValueOnce({ agent: "openclaw" }); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "work", "--", "--json"] }, + { execJson }, + ); + + expect(execJson).not.toHaveBeenCalled(); + expect(execMock).toHaveBeenCalledWith( + "alpha", + ["openclaw", "agent", "--agent", "work", "--", "--json"], + { tty: false }, + ); + }); + + it("keeps --json after an unknown future value flag on the normal passthrough path", async () => { + const execJson = vi.fn(((): never => { + throw new Error("__unexpected-json"); + }) as NonNullable); + getSandboxMock.mockReturnValueOnce({ agent: "openclaw" }); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "work", "--some-future-value-flag", "--json"] }, + { execJson }, + ); + + expect(execJson).not.toHaveBeenCalled(); + expect(execMock).toHaveBeenCalledWith( + "alpha", + ["openclaw", "agent", "--agent", "work", "--some-future-value-flag", "--json"], + { tty: false }, + ); + }); + + it("uses the captured JSON path after documented OpenClaw boolean flags", async () => { + const execJson = vi.fn(() => { + throw new Error("__exit:0"); + }); + getSandboxMock.mockReturnValueOnce({ agent: "openclaw" }); + const { proc } = makeProcMock(); + + await expect( + runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "work", "--deliver", "--json", "-m", "ping"] }, + { execJson, process: proc }, + ), + ).rejects.toThrow("__exit:0"); + + expect(execMock).not.toHaveBeenCalled(); + expect(execJson).toHaveBeenCalledWith( + "alpha", + ["openclaw", "agent", "--agent", "work", "--deliver", "--json", "-m", "ping"], + expect.objectContaining({ stderr: proc.stderr }), + ); + }); + + it("uses the captured JSON path after documented equals-form value flags", async () => { + const execJson = vi.fn(() => { + throw new Error("__exit:0"); + }); + getSandboxMock.mockReturnValueOnce({ agent: "openclaw" }); + const { proc } = makeProcMock(); + + await expect( + runAgentPassthrough( + "alpha", + { extraArgs: ["--session-id=s1", "--json", "-m", "ping"] }, + { execJson, process: proc }, + ), + ).rejects.toThrow("__exit:0"); + + expect(execMock).not.toHaveBeenCalled(); + expect(execJson).toHaveBeenCalledWith( + "alpha", + ["openclaw", "agent", "--session-id=s1", "--json", "-m", "ping"], + expect.objectContaining({ stderr: proc.stderr }), + ); + }); + + it.each([ + ["-a", "--json"], + ["--agent", "--json"], + ["-m", "--json"], + ["--message", "--json"], + ["--model", "--json"], + ["--provider", "--json"], + ["--reply-channel", "--json"], + ["--session-id", "--json"], + ["--session-key", "--json"], + ["--thinking", "--json"], + ["--timeout", "--json"], + ["--to", "--json"], + ])("keeps --json consumed by %s on the normal passthrough path", async (flag, value) => { + const execJson = vi.fn(((): never => { + throw new Error("__unexpected-json"); + }) as NonNullable); + getSandboxMock.mockReturnValueOnce({ agent: "openclaw" }); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--session-id", "s-1", flag, value] }, + { execJson }, + ); + + expect(execJson).not.toHaveBeenCalled(); + expect(execMock).toHaveBeenCalledWith( + "alpha", + ["openclaw", "agent", "--session-id", "s-1", flag, value], { tty: false }, ); }); diff --git a/src/lib/actions/sandbox/agent/passthrough.ts b/src/lib/actions/sandbox/agent/passthrough.ts index 6dd40a7c286..e2d99c6b4ac 100644 --- a/src/lib/actions/sandbox/agent/passthrough.ts +++ b/src/lib/actions/sandbox/agent/passthrough.ts @@ -72,7 +72,9 @@ // manifest-resolution fail-closed paths, quoted manifest command rejection, // the enforced `--no-tty` argv shape, the non-Ready phase recovery path, the // unparseable phase fail-closed path, the OpenClaw no-selector rejection, and -// the `--flag=value` selector-acceptance branch. +// the `--flag=value` selector-acceptance branch, plus the OpenClaw JSON +// captured transport path used to append failure provenance without polluting +// machine-readable stdout. // // Removal conditions: // @@ -92,12 +94,30 @@ import * as registry from "../../../state/registry"; import { execSandbox } from "../exec"; import { ensureLiveSandboxOrExit } from "../gateway-state"; import { hasAgentPassthroughHelpToken, printAgentPassthroughHelp } from "./passthrough-help"; +import { type AgentJsonPassthroughProcess, runAgentJsonPassthrough } from "./passthrough-json"; export { hasAgentPassthroughHelpToken, printAgentPassthroughHelp, } from "./passthrough-help"; +const OPENCLAW_AGENT_VALUE_FLAGS = new Set([ + "-a", + "--agent", + "-m", + "--message", + "--model", + "--provider", + "--reply-channel", + "--session-id", + "--session-key", + "--thinking", + "--timeout", + "--to", +]); + +const OPENCLAW_AGENT_BOOLEAN_FLAGS = new Set(["--deliver"]); + export interface AgentPassthroughOptions { extraArgs?: readonly string[]; } @@ -106,8 +126,10 @@ export interface AgentPassthroughDeps { getSandbox?: typeof registry.getSandbox; ensureLive?: typeof ensureLiveSandboxOrExit; exec?: typeof execSandbox; + execJson?: typeof runAgentJsonPassthrough; process?: { exit(code: number): never; + stdout?: { write(s: string): unknown }; stderr: { write(s: string): unknown }; }; } @@ -271,6 +293,46 @@ function rejectRegistryReadError( return proc.exit(2); } +function requestsOpenClawJsonOutput(extraArgs: readonly string[]): boolean { + // Invalid state: the host wrapper must pick captured JSON transport only for + // the top-level OpenClaw output flag, not for a literal "--json" consumed as + // a value by another OpenClaw option. Source boundary: upstream OpenClaw owns + // the complete argv grammar; NemoClaw mirrors documented flags only to choose + // the host transport path. Unknown options fail conservative to normal + // passthrough, where OpenClaw parses argv itself. Regression tests cover each + // documented value flag, documented equals-form value flags, documented + // boolean flags, unknown flag fallback, and the `--` terminator. Removal + // condition: OpenClaw exposes a machine-readable argv schema or NemoClaw stops + // special-casing the JSON transport path. + let skipNextValue = false; + for (const arg of extraArgs) { + if (skipNextValue) { + skipNextValue = false; + continue; + } + if (arg === "--") return false; + if (arg === "--json") return true; + if (arg.startsWith("--json=")) { + return !["0", "false", "no", "off"].includes(arg.slice("--json=".length).toLowerCase()); + } + if (OPENCLAW_AGENT_VALUE_FLAGS.has(arg)) { + skipNextValue = true; + continue; + } + const equalsIndex = arg.indexOf("="); + if ( + equalsIndex > 0 && + arg.startsWith("--") && + OPENCLAW_AGENT_VALUE_FLAGS.has(arg.slice(0, equalsIndex)) + ) { + continue; + } + if (OPENCLAW_AGENT_BOOLEAN_FLAGS.has(arg)) continue; + if (arg.startsWith("-")) return false; + } + return false; +} + const TARGET_SELECTOR_FLAGS = ["--agent", "--session-id", "--session-key", "--to"] as const; function hasTargetSelector(args: readonly string[]): boolean { @@ -352,6 +414,15 @@ export async function runAgentPassthrough( if (isOpenClawPassthroughCommand(command) && !hasTargetSelector(extraArgs)) { rejectNoTargetSelector(proc); } + if (isOpenClawPassthroughCommand(command) && requestsOpenClawJsonOutput(extraArgs)) { + const execJson = deps.execJson ?? runAgentJsonPassthrough; + execJson(sandboxName, command, { + exit: proc.exit.bind(proc), + stdout: proc.stdout ?? process.stdout, + stderr: proc.stderr, + } satisfies AgentJsonPassthroughProcess); + return; + } const exec = deps.exec ?? execSandbox; await exec(sandboxName, command, { tty: false }); } diff --git a/src/lib/openclaw/agent-json-provenance.test.ts b/src/lib/openclaw/agent-json-provenance.test.ts new file mode 100644 index 00000000000..07aceb13188 --- /dev/null +++ b/src/lib/openclaw/agent-json-provenance.test.ts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { openClawAgentJsonProvenanceLines } from "./agent-json-provenance"; + +describe("openClawAgentJsonProvenanceLines", () => { + it("returns no provenance for plain successful assistant payloads", () => { + expect( + openClawAgentJsonProvenanceLines(JSON.stringify({ result: { payloads: [{ text: "42" }] } })), + ).toEqual([]); + }); + + it("surfaces failed tool results independent of the bare-python trigger", () => { + const lines = openClawAgentJsonProvenanceLines( + JSON.stringify({ + result: { + messages: [ + { + role: "toolResult", + content: [ + { + type: "toolResult", + toolCallId: "call_false", + toolName: "exec", + isError: true, + text: "exec failed: /bin/false exited 1", + }, + ], + }, + ], + payloads: [{ text: "Done." }], + }, + }), + ); + + expect(lines).toEqual([ + "[openclaw provenance] failed tool result (exec call_false): exec failed: /bin/false exited 1", + ]); + }); + + it("strips ANSI, OSC, and control characters from failed tool details", () => { + const hostile = [ + "\x1B[2Jexec failed", + "\x1B]8;;https://example.invalid/phish\x07linked text\x1B]8;;\x07", + "overwrite\rhidden", + "erase\bmark", + "\u0000done", + ].join(" "); + + const lines = openClawAgentJsonProvenanceLines( + JSON.stringify({ + messages: [ + { + role: "toolResult", + toolCallId: "call_hostile", + toolName: "exec", + isError: true, + text: hostile, + }, + ], + }), + ); + + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("exec failed"); + expect(lines[0]).toContain("linked text"); + expect(lines[0]).not.toMatch(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]/u); + expect(lines[0]).not.toContain("https://example.invalid"); + }); + + it("redacts secret-shaped values in failed tool output before stderr provenance", () => { + const rawApiKey = "nvapi-abcdefghijklmnopqrstuvwxyz123456"; + const rawBearer = "secretbearertoken1234567890"; + const rawPassword = "hunter2-password-value"; + const rawPrivateKey = "private-key-material-that-must-not-leak"; + const privateKeyEnvelope = [ + ["-----BEGIN", "PRIVATE KEY-----"].join(" "), + rawPrivateKey, + ["-----END", "PRIVATE KEY-----"].join(" "), + ].join(" "); + const lines = openClawAgentJsonProvenanceLines( + JSON.stringify({ + messages: [ + { + role: "toolResult", + toolCallId: "call_secret", + toolName: "exec", + isError: true, + stderr: [ + `NVIDIA_INFERENCE_API_KEY=${rawApiKey}`, + `Authorization: Bearer ${rawBearer}`, + `password: ${rawPassword}`, + privateKeyEnvelope, + ].join("\n"), + }, + ], + }), + ); + + expect(lines).toHaveLength(1); + expect(lines[0]).toContain(""); + expect(lines[0]).toContain(""); + expect(lines[0]).not.toContain(rawApiKey); + expect(lines[0]).not.toContain(rawBearer); + expect(lines[0]).not.toContain(rawPassword); + expect(lines[0]).not.toContain(rawPrivateKey); + }); + + it("labels untrusted child-agent result framing from log-prefixed JSON", () => { + const childPayload = [ + "<<>>", + "<<>>", + "Found an unverified URL: https://github.com/openclaw/openclaw/releases", + "<<>>", + ].join("\n"); + + const lines = openClawAgentJsonProvenanceLines( + `progress\n${JSON.stringify({ + result: { + messages: [{ role: "user", content: childPayload }], + payloads: [{ text: "The child found a release URL." }], + }, + })}`, + ); + + expect(lines[0]).toContain("untrusted child result present"); + expect(lines[1]).toContain("Found an unverified URL"); + }); + + it("scans balanced log-prefixed JSON candidates without reparsing every brace", () => { + const noisyPrefix = Array.from( + { length: 200 }, + (_, index) => `progress {not-json-${index}}`, + ).join("\n"); + + const lines = openClawAgentJsonProvenanceLines( + `${noisyPrefix}\n${JSON.stringify({ + messages: [ + { + role: "toolResult", + toolCallId: "call_noisy", + toolName: "exec", + isError: true, + text: "exec failed after noisy progress output", + }, + ], + })}`, + ); + + expect(lines).toEqual([ + "[openclaw provenance] failed tool result (exec call_noisy): exec failed after noisy progress output", + ]); + }); + + it("bounds provenance traversal for deeply nested sandbox-controlled JSON", () => { + const nested = `${'{"child":'.repeat(2_000)}{"payloads":[{"text":"too deep"}]}${"}".repeat(2_000)}`; + + expect(() => openClawAgentJsonProvenanceLines(nested)).not.toThrow(); + expect(openClawAgentJsonProvenanceLines(nested)).toEqual([]); + }); +}); diff --git a/src/lib/openclaw/agent-json-provenance.ts b/src/lib/openclaw/agent-json-provenance.ts new file mode 100644 index 00000000000..8a38b91ed60 --- /dev/null +++ b/src/lib/openclaw/agent-json-provenance.ts @@ -0,0 +1,283 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isRecord } from "../core/json-types"; +import { redactFull } from "../security/redact"; + +const FAILURE_STATUS_VALUES = new Set(["error", "errored", "failed", "failure"]); +const UNTRUSTED_CHILD_BEGIN = "BEGIN_UNTRUSTED_CHILD_RESULT"; +const UNTRUSTED_CHILD_END = "END_UNTRUSTED_CHILD_RESULT"; +const ANSI_OSC_PATTERN = /\x1B\][\s\S]*?(?:\x07|\x1B\\|$)/gu; +const ANSI_CSI_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/gu; +const CONTROL_PATTERN = /[\u0000-\u0007\u000B\u000C\u000E-\u001F\u007F-\u009F]/gu; +const PEM_PRIVATE_KEY_PATTERN = + /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/gu; +const SECRET_KV_PATTERN = + /\b([A-Z0-9_.-]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTHORIZATION)[A-Z0-9_.-]*)\s*[:=]\s*["']?[^"'\s;,)]*/giu; +const MAX_PROVENANCE_WALK_NODES = 10_000; +const MAX_PROVENANCE_WALK_DEPTH = 80; + +type WalkEntry = { + depth: number; + value: unknown; +}; + +function snippet(value: string, limit = 300): string { + const squashed = value + .replace(ANSI_OSC_PATTERN, "") + .replace(ANSI_CSI_PATTERN, "") + .replace(/\r|\u0008/gu, "") + .replace(CONTROL_PATTERN, "") + .replace(/\s+/gu, " ") + .trim(); + const redacted = redactProvenanceDetail(squashed); + return redacted.length <= limit ? redacted : `${redacted.slice(0, limit - 3)}...`; +} + +function redactProvenanceDetail(value: string): string { + return redactFull(value.replace(PEM_PRIVATE_KEY_PATTERN, "")).replace( + SECRET_KV_PATTERN, + "$1=", + ); +} + +function strings(value: unknown): string[] { + const result: string[] = []; + const seen = new WeakSet(); + const stack: WalkEntry[] = [{ value, depth: 0 }]; + let visited = 0; + + while (stack.length > 0 && visited < MAX_PROVENANCE_WALK_NODES) { + const entry = stack.pop(); + if (!entry) break; + visited += 1; + if (entry.depth > MAX_PROVENANCE_WALK_DEPTH) continue; + + if (typeof entry.value === "string") { + result.push(entry.value); + continue; + } + + const children = Array.isArray(entry.value) + ? entry.value + : isRecord(entry.value) + ? Object.values(entry.value) + : []; + if (children.length === 0) continue; + + const objectValue = entry.value as object; + if (seen.has(objectValue)) continue; + seen.add(objectValue); + + for (let index = children.length - 1; index >= 0; index -= 1) { + stack.push({ value: children[index], depth: entry.depth + 1 }); + } + } + + return result; +} + +function detailFromValue(value: unknown): string | null { + if (typeof value === "string") return snippet(value); + if (Array.isArray(value) || isRecord(value)) { + const nested = strings(value) + .map((part) => snippet(part)) + .filter(Boolean); + if (nested.length > 0) return snippet(nested.join("; ")); + try { + return snippet(JSON.stringify(value)); + } catch { + return snippet(String(value)); + } + } + if (value === null || value === undefined) return null; + return snippet(String(value)); +} + +function firstDetail(record: Record): string | null { + for (const key of [ + "text", + "content", + "message", + "error", + "stderr", + "stdout", + "output", + "result", + ]) { + if (Object.hasOwn(record, key)) { + const detail = detailFromValue(record[key]); + if (detail) return detail; + } + } + return null; +} + +function normalized(value: unknown): string { + return String(value || "") + .trim() + .toLowerCase() + .replaceAll("_", "-"); +} + +function isToolLike(record: Record): boolean { + const role = normalized(record.role); + const type = normalized(record.type); + return ( + role === "toolresult" || + role === "tool-result" || + type === "toolresult" || + type === "tool-result" || + ["toolCallId", "tool_call_id", "toolName", "tool_name", "tool"].some((key) => + Object.hasOwn(record, key), + ) + ); +} + +function hasFailureStatus(record: Record): boolean { + if (record.isError === true || record.is_error === true) return true; + for (const key of ["status", "state", "finalStatus"]) { + if (FAILURE_STATUS_VALUES.has(normalized(record[key]))) return true; + } + return record.ok === false || record.success === false; +} + +function toolLabel(record: Record): string { + const tool = record.toolName ?? record.tool_name ?? record.name ?? record.tool; + const callId = record.toolCallId ?? record.tool_call_id ?? record.id; + const parts = [tool, callId].map((part) => String(part || "").trim()).filter(Boolean); + return parts.length > 0 ? parts.join(" ") : "unknown tool"; +} + +function toolFailureLine(record: Record): string | null { + if (!isToolLike(record) || !hasFailureStatus(record)) return null; + const detail = firstDetail(record) ?? "no failure detail provided"; + return `[openclaw provenance] failed tool result (${toolLabel(record)}): ${detail}`; +} + +function collectToolFailureProvenance(value: unknown): string[] { + const lines: string[] = []; + const seen = new WeakSet(); + const stack: WalkEntry[] = [{ value, depth: 0 }]; + let visited = 0; + + while (stack.length > 0 && visited < MAX_PROVENANCE_WALK_NODES) { + const entry = stack.pop(); + if (!entry) break; + visited += 1; + if (entry.depth > MAX_PROVENANCE_WALK_DEPTH) continue; + + const children = Array.isArray(entry.value) + ? entry.value + : isRecord(entry.value) + ? Object.values(entry.value) + : []; + if (!Array.isArray(entry.value) && !isRecord(entry.value)) continue; + + const objectValue = entry.value as object; + if (seen.has(objectValue)) continue; + seen.add(objectValue); + + if (isRecord(entry.value)) { + const line = toolFailureLine(entry.value); + if (line) lines.push(line); + } + + for (let index = children.length - 1; index >= 0; index -= 1) { + stack.push({ value: children[index], depth: entry.depth + 1 }); + } + } + return lines; +} + +function untrustedChildExcerpt(value: string): string | null { + const start = value.indexOf(UNTRUSTED_CHILD_BEGIN); + if (start < 0) return null; + let body = value.slice(start + UNTRUSTED_CHILD_BEGIN.length); + const end = body.indexOf(UNTRUSTED_CHILD_END); + if (end >= 0) body = body.slice(0, end); + body = body.replace(/^[<>\s]+|[<>\s]+$/gu, ""); + return body ? snippet(body) : null; +} + +function collectUntrustedChildProvenance(raw: string, docs: unknown[]): string[] { + const candidates = [...docs.flatMap(strings), raw]; + if (!candidates.some((candidate) => candidate.includes(UNTRUSTED_CHILD_BEGIN))) return []; + + const lines = [ + "[openclaw provenance] untrusted child result present; verify child-sourced data before treating it as confirmed.", + ]; + for (const candidate of candidates) { + const excerpt = untrustedChildExcerpt(candidate); + if (excerpt) { + lines.push(`[openclaw provenance] untrusted child excerpt: ${excerpt}`); + break; + } + } + return lines; +} + +function parseLogPrefixedJsonDocs(raw: string): unknown[] { + const docs: unknown[] = []; + let start: number | null = null; + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = 0; index < raw.length; index += 1) { + const char = raw[index]; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') inString = false; + continue; + } + if (depth > 0 && char === '"') inString = true; + else if (char === "{") { + if (depth === 0) start = index; + depth += 1; + } else if (depth > 0 && char === "}") { + depth -= 1; + if (depth === 0 && start !== null) { + try { + const parsed = JSON.parse(raw.slice(start, index + 1)) as unknown; + docs.push(...(Array.isArray(parsed) ? parsed : [parsed])); + } catch { + // Continue scanning for the next balanced candidate object. + } + start = null; + } + } + } + return docs; +} + +function parseOpenClawJsonDocs(raw: string): unknown[] { + try { + const parsed = JSON.parse(raw) as unknown; + return Array.isArray(parsed) ? parsed : [parsed]; + } catch { + // Invalid state: upstream OpenClaw has emitted log-prefixed/non-clean JSON + // framing for `openclaw agent --json`. Source boundary: OpenClaw owns the + // emitter/framing; NemoClaw only consumes the stream to keep provenance + // visible. Source-fix constraint: do not patch or fork OpenClaw from this + // host-wrapper PR. Regression tests cover log-prefixed balanced candidates + // and provenance extraction. Removal condition: supported OpenClaw versions + // guarantee stable clean JSON framing on stdout. + } + + return parseLogPrefixedJsonDocs(raw); +} + +function dedupe(lines: string[]): string[] { + return Array.from(new Set(lines)); +} + +export function openClawAgentJsonProvenanceLines(raw: string): string[] { + const docs = parseOpenClawJsonDocs(raw); + if (docs.length === 0) return []; + return dedupe([ + ...collectUntrustedChildProvenance(raw, docs), + ...docs.flatMap(collectToolFailureProvenance), + ]); +} diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index 13a7e1edd5e..bbcf380d7af 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -35,6 +35,10 @@ function resultText(result: ProcessResult): string { return [result.stdout, result.stderr].filter(Boolean).join("\n"); } +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} + function outputContainsSandbox(result: ProcessResult, sandboxName: string): boolean { const escaped = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return new RegExp(`(^|\\s)${escaped}(\\s|$)`, "m").test(resultText(result)); @@ -44,6 +48,17 @@ function expectExitZero(result: ProcessResult, label: string): void { expect(result.exitCode, `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); } +function expectJsonStdout(result: ProcessResult, label: string): void { + expect( + result.stdout.trim(), + `${label} produced empty stdout\nstderr:\n${result.stderr}`, + ).not.toBe(""); + expect( + () => JSON.parse(result.stdout), + `${label} stdout is not JSON:\n${result.stdout}`, + ).not.toThrow(); +} + async function cleanupSandbox(host: HostCliClient, sandboxName: string): Promise { const result = await host.nemoclaw([sandboxName, "destroy", "--yes"], { artifactName: `cleanup-destroy-${sandboxName}`, @@ -243,13 +258,7 @@ async function assertAgentCanAnswer(host: HostCliClient, sandboxName: string): P const sessionId = `e2e-sbx-02-${Date.now()}-${process.pid}`; const result = await host.nemoclaw( [ - "sandbox", - "exec", sandboxName, - "--timeout", - "90", - "--", - "openclaw", "agent", "--agent", "main", @@ -260,16 +269,79 @@ async function assertAgentCanAnswer(host: HostCliClient, sandboxName: string): P "What is 6 multiplied by 7? Reply with only the integer, no extra words.", ], { - artifactName: "tc-sbx-02-openclaw-agent-json", + artifactName: "tc-sbx-02-nemoclaw-agent-json", env: buildAvailabilityProbeEnv(), timeoutMs: 120_000, }, ); const reply = parseOpenClawAgentText(result.stdout); - expectExitZero(result, "openclaw agent --json"); + expectExitZero(result, `nemoclaw ${sandboxName} agent --json`); expect(containsInteger42Answer(reply), resultText(result)).toBe(true); } +async function assertAgentJsonTransportBoundaries( + host: HostCliClient, + sandboxName: string, +): Promise { + const invalidFlag = await host.nemoclaw( + [sandboxName, "agent", "--json", "--nemoclaw-e2e-invalid-openclaw-agent-flag"], + { + artifactName: "tc-sbx-02b-agent-json-nonzero", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + expect(invalidFlag.timedOut, resultText(invalidFlag)).toBe(false); + expect(invalidFlag.exitCode, resultText(invalidFlag)).not.toBeNull(); + expect(invalidFlag.exitCode, resultText(invalidFlag)).not.toBe(0); + + const stdinPrompt = "What is 6 multiplied by 7? Reply with only the integer, no extra words."; + const stdinSessionId = `e2e-sbx-02b-stdin-${Date.now()}-${process.pid}`; + const stdinScript = [ + "set -euo pipefail", + `printf '%s\\n' ${shellQuote(stdinPrompt)} | ${shellQuote(host.commandPath)} ${shellQuote( + sandboxName, + )} agent --agent main --json --session-id ${shellQuote(stdinSessionId)}`, + ].join("\n"); + const stdinResult = await host.command("bash", ["-lc", stdinScript], { + artifactName: "tc-sbx-02b-agent-json-stdin", + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }); + expectExitZero(stdinResult, `printf prompt | nemoclaw ${sandboxName} agent --json`); + expectJsonStdout(stdinResult, "stdin agent --json"); + expect(parseOpenClawAgentText(stdinResult.stdout), resultText(stdinResult)).toMatch( + /(^|[^0-9])42([^0-9]|$)/, + ); + + const provenanceMarker = `NEMOCLAW_PROVENANCE_E2E_${Date.now()}_${process.pid}`; + const failureSessionId = `e2e-sbx-02b-failure-${Date.now()}-${process.pid}`; + const failure = await host.nemoclaw( + [ + sandboxName, + "agent", + "--agent", + "main", + "--json", + "--thinking", + "off", + "--session-id", + failureSessionId, + "-m", + `Use the exec tool to run exactly: /bin/sh -lc 'echo ${provenanceMarker}; exit 7'. Do not retry the command. Then reply with exactly: observed expected failure.`, + ], + { + artifactName: "tc-sbx-02b-agent-json-failed-tool-provenance", + env: buildAvailabilityProbeEnv(), + timeoutMs: 180_000, + }, + ); + expectExitZero(failure, `nemoclaw ${sandboxName} agent --json failed-tool provenance`); + expectJsonStdout(failure, "failed-tool agent --json"); + expect(failure.stderr, resultText(failure)).toContain("[openclaw provenance] failed tool result"); + expect(failure.stderr, resultText(failure)).toContain(provenanceMarker); +} + async function assertStatusFields(host: HostCliClient, sandboxName: string): Promise { const status = await host.nemoclaw([sandboxName, "status"], { artifactName: "tc-sbx-03-status-fields", @@ -538,7 +610,8 @@ liveTest( legacySource: "test/e2e/test-sandbox-operations.sh", contracts: [ "TC-SBX-01 list shows onboarded sandbox", - "TC-SBX-02 openclaw agent answers through sandbox inference.local", + "TC-SBX-02 nemoclaw agent --json answers through sandbox inference.local", + "TC-SBX-02b agent --json preserves stdin, nonzero status, and failed-tool provenance boundaries", "TC-SBX-03 status renders Sandbox/Model/Provider/GPU fields", "TC-SBX-04 logs and logs --follow behave as streaming commands", "TC-SBX-05 destroy removes NemoClaw and OpenShell entries", @@ -572,6 +645,7 @@ liveTest( await expectListed(host, SANDBOX_A, "tc-sbx-01-list-sandbox-a"); await assertAgentCanAnswer(host, SANDBOX_A); + await assertAgentJsonTransportBoundaries(host, SANDBOX_A); await assertStatusFields(host, SANDBOX_A); await assertLogsStream(host, SANDBOX_A); await assertTmuxPtyFlow(sandbox, SANDBOX_A); diff --git a/test/e2e/lib/openclaw-agent-json.py b/test/e2e/lib/openclaw-agent-json.py index c99e9f5a168..bd045ee80d6 100755 --- a/test/e2e/lib/openclaw-agent-json.py +++ b/test/e2e/lib/openclaw-agent-json.py @@ -2,36 +2,360 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Extract text payloads from `openclaw agent --json` output. +"""Extract user-visible text from `openclaw agent --json` output. OpenClaw has emitted both of these envelopes across recent versions: {"result": {"payloads": [{"text": "..."}]}} {"payloads": [{"text": "..."}]} -The E2E smoke checks only need the joined assistant text. Invalid JSON is a -real harness failure and exits nonzero; valid JSON with no text prints nothing. +The E2E smoke checks usually need the joined assistant text, but tool failures +and untrusted child-agent payloads are also user-visible provenance. Preserve +those markers so a plausible assistant reply cannot hide failed or unverified +work. Invalid JSON is a real harness failure and exits nonzero; valid JSON with +no visible text prints nothing. """ from __future__ import annotations import json +import re import sys from typing import Any +FAILURE_STATUS_VALUES = {"error", "errored", "failed", "failure"} +UNTRUSTED_CHILD_BEGIN = "BEGIN_UNTRUSTED_CHILD_RESULT" +UNTRUSTED_CHILD_END = "END_UNTRUSTED_CHILD_RESULT" +ANSI_OSC_RE = re.compile(r"\x1B\][\s\S]*?(?:\x07|\x1B\\|$)") +ANSI_CSI_RE = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]") +CONTROL_RE = re.compile(r"[\x00-\x07\x0b\x0c\x0e-\x1f\x7f-\x9f]") +PEM_PRIVATE_KEY_RE = re.compile( + r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----" +) +SECRET_PREFIX_RES = ( + re.compile(r"nvapi-[A-Za-z0-9_-]{10,}"), + re.compile(r"nvcf-[A-Za-z0-9_-]{10,}"), + re.compile(r"ghp_[A-Za-z0-9_-]{10,}"), + re.compile(r"github_pat_[A-Za-z0-9_]{30,}"), + re.compile(r"sk-proj-[A-Za-z0-9_-]{10,}"), + re.compile(r"sk-ant-[A-Za-z0-9_-]{10,}"), + re.compile(r"sk-[A-Za-z0-9_-]{20,}"), + re.compile(r"(?:xox[bpas]|xapp)-[A-Za-z0-9-]{10,}"), + re.compile(r"A(?:K|S)IA[A-Z0-9]{16}"), + re.compile(r"hf_[A-Za-z0-9]{10,}"), + re.compile(r"glpat-[A-Za-z0-9_-]{10,}"), + re.compile(r"gsk_[A-Za-z0-9]{10,}"), + re.compile(r"pypi-[A-Za-z0-9_-]{10,}"), + re.compile(r"\bbot\d{8,10}:[A-Za-z0-9_-]{35}\b"), + re.compile(r"\b\d{8,10}:[A-Za-z0-9_-]{35}\b"), + re.compile(r"\b[A-Za-z0-9]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}\b"), +) +BEARER_RE = re.compile(r"(Bearer\s+)\S+", re.IGNORECASE) +SECRET_KV_RE = re.compile( + r"\b([A-Z0-9_.-]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTHORIZATION)[A-Z0-9_.-]*\s*[:=]\s*[\"']?)[^\"'\s;,)]*", + re.IGNORECASE, +) +TEXT_KEYS = ("text", "content", "reasoning_content", "reasoning") +CONTAINER_KEYS = ( + "result", + "payloads", + "payload", + "messages", + "message", + "response", + "data", + "output", + "outputs", + "items", + "segments", + "delta", +) +MAX_HELPER_WALK_NODES = 10_000 +MAX_HELPER_WALK_DEPTH = 80 -def _payloads(doc: Any) -> list[Any]: - if not isinstance(doc, dict): + +def _new_walk_budget() -> dict[str, Any]: + return {"nodes": 0, "seen": set()} + + +def _can_visit(value: Any, depth: int, budget: dict[str, Any]) -> bool: + if depth > MAX_HELPER_WALK_DEPTH: + return False + if budget["nodes"] >= MAX_HELPER_WALK_NODES: + return False + budget["nodes"] += 1 + if isinstance(value, (dict, list)): + marker = id(value) + if marker in budget["seen"]: + return False + budget["seen"].add(marker) + return True + + +def _snippet(value: str, limit: int = 300) -> str: + sanitized = ANSI_OSC_RE.sub("", value) + sanitized = ANSI_CSI_RE.sub("", sanitized) + sanitized = sanitized.replace("\r", "").replace("\b", "") + sanitized = CONTROL_RE.sub("", sanitized) + squashed = re.sub(r"\s+", " ", sanitized).strip() + redacted = _redact_secret_text(squashed) + if len(redacted) <= limit: + return redacted + return f"{redacted[: limit - 3]}..." + + +def _redact_secret_text(value: str) -> str: + redacted = PEM_PRIVATE_KEY_RE.sub("", value) + redacted = BEARER_RE.sub(r"\1", redacted) + redacted = SECRET_KV_RE.sub(r"\1", redacted) + for pattern in SECRET_PREFIX_RES: + redacted = pattern.sub("", redacted) + return redacted + + +def _strings(value: Any, depth: int = 0, budget: dict[str, Any] | None = None) -> list[str]: + if budget is None: + budget = _new_walk_budget() + if not _can_visit(value, depth, budget): return [] - top_level = doc.get("payloads") - if isinstance(top_level, list): - return top_level - result = doc.get("result") - if isinstance(result, dict) and isinstance(result.get("payloads"), list): - return result["payloads"] + if isinstance(value, str): + return [value] + if isinstance(value, list): + parts: list[str] = [] + for item in value: + parts.extend(_strings(item, depth + 1, budget)) + return parts + if isinstance(value, dict): + parts = [] + for item in value.values(): + parts.extend(_strings(item, depth + 1, budget)) + return parts return [] +def _detail_from_value(value: Any) -> str | None: + if isinstance(value, str): + return _snippet(value) + if isinstance(value, (dict, list)): + strings = [_snippet(part) for part in _strings(value) if part.strip()] + if strings: + return _snippet("; ".join(strings)) + try: + return _snippet(json.dumps(value, sort_keys=True)) + except TypeError: + return _snippet(str(value)) + if value is None: + return None + return _snippet(str(value)) + + +def _first_detail(record: dict[str, Any]) -> str | None: + for key in ( + "text", + "content", + "message", + "error", + "stderr", + "stdout", + "output", + "result", + ): + if key in record: + detail = _detail_from_value(record[key]) + if detail: + return detail + return None + + +def _normalized(value: Any) -> str: + return str(value or "").strip().lower().replace("_", "-") + + +def _is_tool_like(record: dict[str, Any]) -> bool: + role = _normalized(record.get("role")) + block_type = _normalized(record.get("type")) + if role == "toolresult" or block_type == "toolresult": + return True + if role == "tool-result" or block_type == "tool-result": + return True + return any( + key in record + for key in ( + "toolCallId", + "tool_call_id", + "toolName", + "tool_name", + "tool", + ) + ) + + +def _has_failure_status(record: dict[str, Any]) -> bool: + if record.get("isError") is True or record.get("is_error") is True: + return True + for key in ("status", "state", "finalStatus"): + if _normalized(record.get(key)) in FAILURE_STATUS_VALUES: + return True + return record.get("ok") is False or record.get("success") is False + + +def _tool_label(record: dict[str, Any]) -> str: + tool = ( + record.get("toolName") + or record.get("tool_name") + or record.get("name") + or record.get("tool") + ) + call_id = record.get("toolCallId") or record.get("tool_call_id") or record.get("id") + parts = [str(part).strip() for part in (tool, call_id) if str(part or "").strip()] + return " ".join(parts) if parts else "unknown tool" + + +def _tool_failure_line(record: dict[str, Any]) -> str | None: + if not _is_tool_like(record) or not _has_failure_status(record): + return None + detail = _first_detail(record) or "no failure detail provided" + return f"[openclaw provenance] failed tool result ({_tool_label(record)}): {detail}" + + +def _collect_tool_failure_provenance(value: Any) -> list[str]: + lines: list[str] = [] + + def visit(node: Any, depth: int, budget: dict[str, Any]) -> None: + if not _can_visit(node, depth, budget): + return + if isinstance(node, dict): + line = _tool_failure_line(node) + if line: + lines.append(line) + for child in node.values(): + visit(child, depth + 1, budget) + elif isinstance(node, list): + for child in node: + visit(child, depth + 1, budget) + + visit(value, 0, _new_walk_budget()) + return lines + + +def _untrusted_child_excerpt(value: str) -> str | None: + start = value.find(UNTRUSTED_CHILD_BEGIN) + if start < 0: + return None + body = value[start + len(UNTRUSTED_CHILD_BEGIN) :] + end = body.find(UNTRUSTED_CHILD_END) + if end >= 0: + body = body[:end] + body = body.strip(" <>\n\r\t") + return _snippet(body) if body else None + + +def _collect_untrusted_child_provenance(raw: str, docs: list[Any]) -> list[str]: + candidates: list[str] = [] + budget = _new_walk_budget() + for doc in docs: + candidates.extend(_strings(doc, budget=budget)) + candidates.append(raw) + if not any(UNTRUSTED_CHILD_BEGIN in candidate for candidate in candidates): + return [] + + lines = [ + "[openclaw provenance] untrusted child result present; verify child-sourced data before treating it as confirmed." + ] + for candidate in candidates: + excerpt = _untrusted_child_excerpt(candidate) + if excerpt: + lines.append(f"[openclaw provenance] untrusted child excerpt: {excerpt}") + break + return lines + + +def _dedupe(lines: list[str]) -> list[str]: + seen: set[str] = set() + result: list[str] = [] + for line in lines: + if line in seen: + continue + seen.add(line) + result.append(line) + return result + + +def _collect_provenance(raw: str, docs: list[Any]) -> list[str]: + lines: list[str] = [] + lines.extend(_collect_untrusted_child_provenance(raw, docs)) + for doc in docs: + lines.extend(_collect_tool_failure_provenance(doc)) + return _dedupe(lines) + + +def _add_text(parts: list[str], value: Any) -> None: + if isinstance(value, str) and value.strip(): + parts.append(value.strip()) + + +def _collect_assistant_text( + value: Any, + parts: list[str], + visited: set[int], + depth: int = 0, + budget: dict[str, Any] | None = None, +) -> None: + if budget is None: + budget = _new_walk_budget() + if not _can_visit(value, depth, budget): + return + if isinstance(value, str): + _add_text(parts, value) + return + if isinstance(value, list): + marker = id(value) + if marker in visited: + return + visited.add(marker) + for item in value: + _collect_assistant_text(item, parts, visited, depth + 1, budget) + return + if not isinstance(value, dict): + return + + marker = id(value) + if marker in visited: + return + visited.add(marker) + + if _is_tool_like(value): + return + + for key in TEXT_KEYS: + _add_text(parts, value.get(key)) + + choices = value.get("choices") + if isinstance(choices, list): + for choice in choices: + if not isinstance(choice, dict): + continue + _collect_assistant_text(choice.get("message"), parts, visited, depth + 1, budget) + _collect_assistant_text(choice.get("delta"), parts, visited, depth + 1, budget) + _add_text(parts, choice.get("text")) + + for key in CONTAINER_KEYS: + if key in value: + _collect_assistant_text(value[key], parts, visited, depth + 1, budget) + + +def _assistant_text_parts(docs: list[Any]) -> list[str]: + parts: list[str] = [] + visited: set[int] = set() + budget = _new_walk_budget() + for doc in docs: + if isinstance(doc, dict) and "result" in doc: + _collect_assistant_text(doc["result"], parts, visited, budget=budget) + else: + _collect_assistant_text(doc, parts, visited, budget=budget) + return parts + + def _load_agent_json_docs(text: str) -> list[Any]: try: doc = json.loads(text) @@ -40,6 +364,16 @@ def _load_agent_json_docs(text: str) -> list[Any]: else: return doc if isinstance(doc, list) else [doc] + # Invalid state: upstream OpenClaw has emitted log-prefixed/non-clean JSON + # framing for `openclaw agent --json`, and sandbox-controlled JSON can be + # deeply nested. Source boundary: OpenClaw owns the emitter/framing; the host + # TypeScript parser owns CLI provenance extraction; this E2E helper only + # preserves legacy smoke-test text/provenance assertions. Source-fix + # constraint: do not patch OpenClaw or broaden production parser callers from + # this PR. Regression tests cover log-prefixed streams, later envelopes, + # OpenAI-style choices, sanitized provenance, and bounded deep traversal. + # Removal condition: supported OpenClaw versions guarantee stable clean JSON + # framing on stdout or these shell smoke tests use the host TS parser. decoder = json.JSONDecoder() docs: list[Any] = [] index = 0 @@ -63,17 +397,12 @@ def main() -> int: raw = sys.stdin.read() try: docs = _load_agent_json_docs(raw) - except json.JSONDecodeError as err: + except (json.JSONDecodeError, RecursionError) as err: print(f"invalid JSON: {err}", file=sys.stderr) return 1 - parts = [ - payload["text"] - for doc in docs - for payload in _payloads(doc) - if isinstance(payload, dict) and isinstance(payload.get("text"), str) - ] - print("\n".join(parts)) + parts = _assistant_text_parts(docs) + print("\n".join([*_collect_provenance(raw, docs), *parts])) return 0 diff --git a/test/e2e/lib/openclaw-json.sh b/test/e2e/lib/openclaw-json.sh index 95c8eb067a2..0b104a72c2d 100755 --- a/test/e2e/lib/openclaw-json.sh +++ b/test/e2e/lib/openclaw-json.sh @@ -4,10 +4,12 @@ # Extract human-readable assistant text from `openclaw agent --json` output. # OpenClaw's JSON envelope has moved between result.payloads[] and top-level -# payloads[]; keep E2E assertions focused on visible reply text instead of one -# exact envelope shape. This also tolerates wrapper output before the JSON blob -# but intentionally ignores metadata fields so IDs, durations, session names, -# and model/provider details cannot satisfy reply assertions. +# payloads[]; keep E2E assertions focused on visible reply/provenance text +# instead of one exact envelope shape. This also tolerates wrapper output before +# the JSON blob while preserving failed-tool and untrusted-child provenance so +# plausible assistant text cannot hide incomplete or unverified work. Metadata +# fields such as IDs, durations, session names, and model/provider details +# should not satisfy reply assertions. e2e_text_contains_integer_42() { local compact compact="$(printf '%s' "${1:-}" | tr -d '[:space:]')" @@ -15,84 +17,9 @@ e2e_text_contains_integer_42() { } parse_openclaw_agent_text() { - python3 -c ' -import json -import sys - -raw = sys.stdin.read() -if not raw.strip(): - sys.exit(0) - -parts = [] -visited = set() - -TEXT_KEYS = {"text", "content", "reasoning_content"} -CONTAINER_KEYS = { - "result", "payloads", "payload", "messages", "choices", "response", - "data", "output", "outputs", "items", "segments", "delta", -} - - -def add(value): - if isinstance(value, str) and value.strip(): - parts.append(value.strip()) - - -def collect(value): - value_id = id(value) - if value_id in visited: - return - visited.add(value_id) - - if isinstance(value, str): - add(value) - return - if isinstance(value, list): - for item in value: - collect(item) - return - if not isinstance(value, dict): - return - - for key in TEXT_KEYS: - add(value.get(key)) - - # OpenAI-style choices can nest assistant text under message/delta objects. - for choice in value.get("choices") or []: - if isinstance(choice, dict): - collect(choice.get("message")) - collect(choice.get("delta")) - add(choice.get("text")) - - for key in CONTAINER_KEYS: - if key in value: - collect(value[key]) - - -def collect_from_doc(doc): - if isinstance(doc, dict) and isinstance(doc.get("result"), dict): - collect(doc["result"]) - else: - collect(doc) - -try: - collect_from_doc(json.loads(raw)) -except Exception: - decoder = json.JSONDecoder() - for idx, char in enumerate(raw): - if char != "{": - continue - try: - doc, _end = decoder.raw_decode(raw[idx:]) - except Exception: - continue - before = len(parts) - collect_from_doc(doc) - if len(parts) > before: - break - -print("\n".join(parts)) -' + local helper_dir + helper_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + python3 "${helper_dir}/openclaw-agent-json.py" } nemoclaw_e2e_compact_agent_reply() { diff --git a/test/openclaw-agent-json.test.ts b/test/openclaw-agent-json.test.ts index 8ea7d21a05d..693c3cfb6b3 100644 --- a/test/openclaw-agent-json.test.ts +++ b/test/openclaw-agent-json.test.ts @@ -1,14 +1,15 @@ -// @ts-nocheck // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; -import path from "node:path"; +import type { SpawnSyncReturns } from "node:child_process"; import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; const HELPER = path.join(import.meta.dirname, "e2e", "lib", "openclaw-agent-json.py"); +const SHELL_HELPER = path.join(import.meta.dirname, "e2e", "lib", "openclaw-json.sh"); -function runHelper(input: string) { +function runHelper(input: string): SpawnSyncReturns { return spawnSync("python3", [HELPER], { input, encoding: "utf-8", @@ -16,6 +17,15 @@ function runHelper(input: string) { }); } +function runShellHelper(input: string): SpawnSyncReturns { + return spawnSync("bash", ["-lc", 'source "$OPENCLAW_JSON_HELPER"; parse_openclaw_agent_text'], { + input, + encoding: "utf-8", + env: { ...process.env, OPENCLAW_JSON_HELPER: SHELL_HELPER }, + stdio: ["pipe", "pipe", "pipe"], + }); +} + describe("openclaw-agent-json.py", () => { it("extracts nested result payload text", () => { const result = runHelper(JSON.stringify({ result: { payloads: [{ text: "42" }] } })); @@ -61,4 +71,195 @@ describe("openclaw-agent-json.py", () => { expect(result.status).toBe(0); expect(result.stdout).toBe("42\n"); }); + + it("preserves failed tool-result provenance independent of missing bare python", () => { + const result = runHelper( + JSON.stringify({ + result: { + messages: [ + { + role: "toolResult", + content: [ + { + type: "toolResult", + toolCallId: "call_node_missing", + toolName: "exec", + isError: true, + text: "exec failed: /bin/sh: 1: node-not-a-real-command: not found", + }, + ], + }, + ], + payloads: [{ text: "The script was saved successfully." }], + }, + }), + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("[openclaw provenance] failed tool result"); + expect(result.stdout).toContain("exec call_node_missing"); + expect(result.stdout).toContain("node-not-a-real-command"); + expect(result.stdout).toContain("The script was saved successfully."); + expect(result.stdout.indexOf("[openclaw provenance] failed tool result")).toBeLessThan( + result.stdout.indexOf("The script was saved successfully."), + ); + }); + + it("strips ANSI, OSC, and control characters from provenance details", () => { + const result = runHelper( + JSON.stringify({ + messages: [ + { + role: "toolResult", + toolCallId: "call_hostile", + toolName: "exec", + isError: true, + text: [ + "\x1B[2Jexec failed", + "\x1B]8;;https://example.invalid/phish\x07linked text\x1B]8;;\x07", + "overwrite\rhidden", + "erase\bmark", + "\u0000done", + ].join(" "), + }, + ], + }), + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("[openclaw provenance] failed tool result"); + expect(result.stdout).toContain("exec failed"); + expect(result.stdout).toContain("linked text"); + expect(result.stdout).not.toMatch(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]/u); + expect(result.stdout).not.toContain("https://example.invalid"); + }); + + it("redacts secret-shaped values from provenance details", () => { + const rawApiKey = "nvapi-abcdefghijklmnopqrstuvwxyz123456"; + const rawBearer = "secretbearertoken1234567890"; + const rawPassword = "hunter2-password-value"; + const rawPrivateKey = "private-key-material-that-must-not-leak"; + const privateKeyEnvelope = [ + ["-----BEGIN", "PRIVATE KEY-----"].join(" "), + rawPrivateKey, + ["-----END", "PRIVATE KEY-----"].join(" "), + ].join(" "); + const result = runHelper( + JSON.stringify({ + messages: [ + { + role: "toolResult", + toolCallId: "call_secret", + toolName: "exec", + isError: true, + stderr: [ + `NVIDIA_INFERENCE_API_KEY=${rawApiKey}`, + `Authorization: Bearer ${rawBearer}`, + `password: ${rawPassword}`, + privateKeyEnvelope, + ].join("\n"), + }, + ], + }), + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("[openclaw provenance] failed tool result"); + expect(result.stdout).toContain(""); + expect(result.stdout).toContain(""); + expect(result.stdout).not.toContain(rawApiKey); + expect(result.stdout).not.toContain(rawBearer); + expect(result.stdout).not.toContain(rawPassword); + expect(result.stdout).not.toContain(rawPrivateKey); + }); + + it("bounds deep traversal of sandbox-controlled JSON", () => { + let envelope: unknown = { payloads: [{ text: "too deep to trust" }] }; + for (let index = 0; index < 120; index += 1) { + envelope = { payload: envelope }; + } + + const result = runHelper(JSON.stringify(envelope)); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toBe("\n"); + }); + + it("preserves legacy assistant response shapes from choices and nested containers", () => { + const result = runHelper( + JSON.stringify([ + { + choices: [ + { message: { content: "choice message" } }, + { delta: { content: "delta chunk" } }, + { text: "choice text" }, + ], + }, + { response: { reasoning_content: "reasoning output" } }, + { result: { messages: [{ content: "nested message content" }] } }, + ]), + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("choice message"); + expect(result.stdout).toContain("delta chunk"); + expect(result.stdout).toContain("choice text"); + expect(result.stdout).toContain("reasoning output"); + expect(result.stdout).toContain("nested message content"); + }); + + it("labels untrusted child-agent payloads before assistant text", () => { + const childPayload = [ + "<<>>", + "<<>>", + "Found a plausible but unverified URL: https://github.com/openclaw/openclaw/releases", + "<<>>", + ].join("\n"); + const result = runHelper( + JSON.stringify({ + result: { + messages: [{ role: "user", content: childPayload }], + payloads: [ + { + text: "The web-search skill found https://github.com/openclaw/openclaw/releases.", + }, + ], + }, + }), + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("[openclaw provenance] untrusted child result present"); + expect(result.stdout).toContain("unverified URL"); + expect(result.stdout).toContain("The web-search skill found"); + expect( + result.stdout.indexOf("[openclaw provenance] untrusted child result present"), + ).toBeLessThan(result.stdout.indexOf("The web-search skill found")); + }); + + it("routes the shell E2E parser through the provenance-preserving helper", () => { + const result = runShellHelper( + JSON.stringify({ + payloads: [{ text: "Finished." }], + messages: [ + { + role: "toolResult", + toolCallId: "call_false", + toolName: "exec", + isError: true, + text: "exec failed: /bin/false exited 1", + }, + ], + }), + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("[openclaw provenance] failed tool result"); + expect(result.stdout).toContain("/bin/false exited 1"); + expect(result.stdout).toContain("Finished."); + expect(result.stdout.indexOf("[openclaw provenance] failed tool result")).toBeLessThan( + result.stdout.indexOf("Finished."), + ); + }); });