diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index dfd0a166d64..9f1f61dd055 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1674,6 +1674,38 @@ The classifications are `blocked-by-policy`, `missing-approval`, `unsupported`, | `--write` | Refresh `/sandbox/.openclaw/workspace/POLICY.md` inside the sandbox in addition to printing | +### `$$nemoclaw policy-simulate` + +Dry-run a policy against a recorded agent execution trace without changing the sandbox. +The command evaluates each request in the trace against the sandbox's active policy presets, or against a candidate policy file, and reports which requests would be allowed, blocked, or not covered by any preset. +It exits non-zero when any request would be blocked or uncovered, so automation can gate a policy change on the result. + +Trace files are JSONL with one JSON object per line. +Only the `host` field is required; missing `port`, `method`, and `path` fields match any endpoint rule for that host. + +```bash +$$nemoclaw my-assistant policy-simulate --from-file ./agent-trace.jsonl +``` + +Preview a candidate policy file before applying it with `policy-add`: + +```bash +$$nemoclaw my-assistant policy-simulate --policy-file ./presets/my-api.yaml --from-file ./agent-trace.jsonl +``` + +Pipe requests from another process by passing `-` as the trace file: + +```bash +cat trace.jsonl | $$nemoclaw my-assistant policy-simulate --from-file - +``` + +| Flag | Description | +|------|-------------| +| `--from-file` | Path to a JSONL trace file, or `-` to read the trace from stdin | +| `--policy-file` | Simulate a candidate policy YAML file instead of the active sandbox policy | +| `--preset-name` | Name to assign to the candidate presets when `--policy-file` is used | +| `--json` | Emit the simulation results as a structured JSON object | + ### `$$nemoclaw hosts-add` Add a host alias to the sandbox pod template. diff --git a/src/commands/sandbox/policy/simulate.ts b/src/commands/sandbox/policy/simulate.ts new file mode 100644 index 00000000000..3517921ef25 --- /dev/null +++ b/src/commands/sandbox/policy/simulate.ts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import readline from "node:readline"; + +import { Args, Flags } from "@oclif/core"; + +import { simulateSandboxPolicy } from "../../../lib/actions/sandbox/policy-simulate"; +import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; +import { renderSimulationReport, summaryNeedsAttention } from "../../../lib/policy/simulate"; + +export default class PolicySimulateCommand extends NemoClawCommand { + static id = "sandbox:policy:simulate"; + static strict = true; + static summary = "Statically evaluate a recorded trace against registered policy content"; + static description = + `Statically evaluate which network requests in a recorded trace match the sandbox's registered policy content (built-in presets plus custom and generated policies), or a candidate policy file. + +The evaluation is fail-closed and covers host, port, method, and path only. A request is reported ALLOWED only when every evaluated dimension is proven from the trace row; policy constraints the engine does not evaluate (protocol, allowed_ips, TLS, ancestry, MCP, deny rules) and trace rows missing a constrained field produce UNKNOWN verdicts. Malformed trace rows are reported, not dropped. Live gateway state is not consulted, so drift between the registry and the gateway is not detected. + +Trace files are JSONL, one JSON object per line with at minimum a "host" field: + {"host":"api.slack.com","port":443,"method":"POST","path":"/api/chat.postMessage"} + +Use --from-file to provide a recorded trace, or pipe requests to stdin. +Use --policy-file to evaluate a candidate policy YAML without applying it. +Exit code is non-zero when any request is blocked, uncovered, or unknown, or any trace row is invalid.`; + + static usage = [" --from-file [--policy-file ] [--json]"]; + + static examples = [ + "<%= config.bin %> sandbox policy simulate alpha --from-file ./agent-trace.jsonl", + "<%= config.bin %> sandbox policy simulate alpha --policy-file ./slack.yaml --from-file ./trace.jsonl", + "<%= config.bin %> sandbox policy simulate alpha --from-file ./trace.jsonl --json", + ]; + + static args = { + sandboxName: Args.string({ + name: "sandbox", + description: "Sandbox name", + ignoreStdin: true, + required: true, + }), + }; + + static flags = { + "from-file": Flags.string({ + description: + 'Path to a JSONL trace file, or "-" to read from stdin. Each line: {"host":"...","port":443,"method":"GET","path":"/"}', + required: true, + }), + "policy-file": Flags.string({ + description: + "Path to a candidate policy YAML file to test instead of the active sandbox policy. Useful for previewing a policy before applying it.", + required: false, + }), + "preset-name": Flags.string({ + description: + "Name to assign to the candidate presets when --policy-file is used. When omitted, preset names from the YAML file are kept.", + required: false, + }), + json: Flags.boolean({ + description: "Output simulation results as JSON", + default: false, + }), + }; + + public async run(): Promise { + const { args, flags } = await this.parse(PolicySimulateCommand); + + const stdinLines = flags["from-file"] === "-" ? await readStdin() : undefined; + + const result = simulateSandboxPolicy({ + sandboxName: args.sandboxName, + fromFile: flags["from-file"], + policyFile: flags["policy-file"], + presetName: flags["preset-name"], + stdinLines, + }); + + if (result.kind === "error") { + this.failWithLines(result.lines); + return; + } + + const report = flags.json + ? JSON.stringify({ ...result.summary, notes: result.notes }, null, 2) + : renderSimulationReport(result.summary, false); + process.stdout.write(report + (report.endsWith("\n") ? "" : "\n")); + if (!flags.json) { + for (const note of result.notes) { + console.error(` Note: ${note}`); + } + } + + if (summaryNeedsAttention(result.summary)) { + this.setExitCode(1); + } + } +} + +async function readStdin(): Promise { + return new Promise((resolve) => { + const lines: string[] = []; + if (process.stdin.isTTY) { + resolve([]); + return; + } + const rl = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); + rl.on("line", (line) => lines.push(line)); + rl.on("close", () => resolve(lines)); + }); +} diff --git a/src/lib/actions/sandbox/policy-simulate.test.ts b/src/lib/actions/sandbox/policy-simulate.test.ts new file mode 100644 index 00000000000..c864b9ff0f1 --- /dev/null +++ b/src/lib/actions/sandbox/policy-simulate.test.ts @@ -0,0 +1,268 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { SimulationSummary } from "../../policy/simulate"; +import { type SimulatePolicyResult, simulateSandboxPolicy } from "./policy-simulate"; + +const SLACK_PRESET = { + name: "slack", + endpoints: [ + { + host: "*.slack.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "POST", path: "/**" } }], + }, + ], +}; + +const POST_TO_SLACK = '{"host":"api.slack.com","port":443,"method":"POST","path":"/api/x"}'; + +const CUSTOM_GITHUB_POLICY_YAML = [ + "network_policies:", + " generated:", + " endpoints:", + " - host: api.github.com", + " port: 443", + " rules:", + " - allow: { method: GET, path: '/**' }", +].join("\n"); + +function expectOk(result: SimulatePolicyResult): { + summary: SimulationSummary; + notes: string[]; +} { + expect(result.kind).toBe("ok"); + return result as { kind: "ok"; summary: SimulationSummary; notes: string[] }; +} + +function expectError(result: SimulatePolicyResult): string[] { + expect(result.kind).toBe("error"); + return (result as { kind: "error"; lines: string[] }).lines; +} + +function throwBadYaml(): never { + throw new Error("bad yaml"); +} + +describe("simulateSandboxPolicy", () => { + it("returns error when trace file does not exist", () => { + const result = simulateSandboxPolicy( + { sandboxName: "alpha", fromFile: "/missing/trace.jsonl" }, + { fileExists: () => false }, + ); + const lines = expectError(result); + expect(lines[0]).toContain("Trace file not found"); + }); + + it("returns error when the trace contains no rows at all", () => { + const result = simulateSandboxPolicy( + { sandboxName: "alpha", fromFile: "-", stdinLines: ["", "# only-a-comment"] }, + { fileExists: () => true }, + ); + const lines = expectError(result); + expect(lines[0]).toContain("No trace requests found"); + }); + + it("evaluates a trace whose only rows are invalid instead of dropping them", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: ["not-json"], + policyFile: "/policies/slack.yaml", + }, + { + fileExists: () => true, + loadPolicy: () => [SLACK_PRESET], + }, + ); + const { summary } = expectOk(result); + expect(summary.totalRequests).toBe(0); + expect(summary.invalidTraceLines).toHaveLength(1); + expect(summary.invalidTraceLines[0].reason).toBe("not valid JSON"); + }); + + it("returns a clean error when the trace file read fails", () => { + const result = simulateSandboxPolicy( + { sandboxName: "alpha", fromFile: "/traces/gone.jsonl" }, + { + fileExists: () => true, + loadTrace: () => { + throw new Error("EACCES: permission denied"); + }, + }, + ); + const lines = expectError(result); + expect(lines[0]).toContain("Failed to read trace file"); + expect(lines[0]).toContain("EACCES"); + }); + + it("returns error when candidate policy file is missing", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: [POST_TO_SLACK], + policyFile: "/missing/policy.yaml", + }, + { fileExists: (p) => p !== "/missing/policy.yaml" }, + ); + const lines = expectError(result); + expect(lines[0]).toContain("Policy file not found"); + }); + + it("returns a clean error when the candidate policy file has invalid YAML", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: [POST_TO_SLACK], + policyFile: "/policies/broken.yaml", + }, + { + fileExists: () => true, + loadPolicy: () => throwBadYaml(), + }, + ); + const lines = expectError(result); + expect(lines[0]).toContain("Failed to parse policy file"); + expect(lines[0]).toContain("bad yaml"); + }); + + it("simulates against a candidate policy file and notes candidate mode", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: [POST_TO_SLACK], + policyFile: "/policies/slack.yaml", + }, + { + fileExists: () => true, + loadPolicy: () => [SLACK_PRESET], + }, + ); + const { summary, notes } = expectOk(result); + expect(summary.allowed).toBe(1); + expect(summary.results[0].allowedBy).toBe("slack"); + expect(notes.join(" ")).toContain("Candidate mode"); + }); + + it("applies presetName override to candidate presets", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: [POST_TO_SLACK], + policyFile: "/policies/slack.yaml", + presetName: "candidate", + }, + { + fileExists: () => true, + loadPolicy: () => [SLACK_PRESET], + }, + ); + const { summary } = expectOk(result); + expect(summary.results[0].allowedBy).toBe("candidate"); + }); + + it("loads active sandbox presets from the registry when no policy file given", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: [POST_TO_SLACK], + }, + { + fileExists: () => true, + loadPolicy: () => [SLACK_PRESET], + getSandboxPolicies: (name) => (name === "alpha" ? ["slack"] : []), + getCustomPolicies: () => [], + presetsDir: "/presets", + }, + ); + const { summary } = expectOk(result); + expect(summary.allowed).toBe(1); + }); + + it("includes custom and generated policies registered on the sandbox (#6269 review)", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: ['{"host":"api.github.com","port":443,"method":"GET","path":"/repos"}'], + }, + { + fileExists: () => false, + getSandboxPolicies: () => [], + getCustomPolicies: () => [ + { name: "mcp-bridge-github", content: CUSTOM_GITHUB_POLICY_YAML }, + ], + presetsDir: "/presets", + }, + ); + const { summary } = expectOk(result); + expect(summary.allowed).toBe(1); + expect(summary.results[0].allowedBy).toBe("mcp-bridge-github"); + }); + + it("notes an applied preset whose file is missing instead of silently skipping it", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: [POST_TO_SLACK], + }, + { + fileExists: () => false, + getSandboxPolicies: () => ["slack"], + getCustomPolicies: () => [ + { name: "mcp-bridge-github", content: CUSTOM_GITHUB_POLICY_YAML }, + ], + presetsDir: "/presets", + }, + ); + const { notes } = expectOk(result); + expect(notes.join(" ")).toContain("Applied preset 'slack' has no readable file"); + expect(notes.join(" ")).toContain("under-report"); + }); + + it("notes an unparseable custom policy instead of silently skipping it", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: [POST_TO_SLACK], + }, + { + fileExists: () => true, + loadPolicy: () => [SLACK_PRESET], + getSandboxPolicies: () => ["slack"], + getCustomPolicies: () => [{ name: "broken-custom", content: "a: [unclosed" }], + presetsDir: "/presets", + }, + ); + const { notes } = expectOk(result); + expect(notes.join(" ")).toContain("Custom policy 'broken-custom' could not be parsed"); + }); + + it("returns error when the sandbox has no registered policy content", () => { + const result = simulateSandboxPolicy( + { + sandboxName: "alpha", + fromFile: "-", + stdinLines: ['{"host":"api.slack.com","port":443}'], + }, + { + fileExists: () => true, + getSandboxPolicies: () => [], + getCustomPolicies: () => [], + }, + ); + const lines = expectError(result); + expect(lines[0]).toContain("No registered policy content"); + }); +}); diff --git a/src/lib/actions/sandbox/policy-simulate.ts b/src/lib/actions/sandbox/policy-simulate.ts new file mode 100644 index 00000000000..bffec3bea2c --- /dev/null +++ b/src/lib/actions/sandbox/policy-simulate.ts @@ -0,0 +1,229 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Host-side orchestration for `sandbox policy simulate`. Loads the trace + * and the policy under test (either a candidate YAML file or the sandbox's + * registered policy set — built-in presets plus custom/generated policies), + * runs the pure simulation engine from `src/lib/policy/simulate`, and + * returns a typed result the command can render. Registry and filesystem + * access live here — behind injectable deps — so the command class stays a + * thin argv adapter. + * + * The evaluation is static: it reflects the policy content the registry + * records, not the live gateway. Sources that cannot be loaded are surfaced + * as notes instead of being silently skipped — a missing allow-source can + * only make results more conservative (never over-claim an allow). + */ + +import fs from "node:fs"; +import path from "node:path"; + +import { + type ParsedPreset, + type ParsedTrace, + parsePolicyContent, + parseTraceLines, + simulate, + type SimulationSummary, +} from "../../policy/simulate"; +import { ROOT } from "../../runner"; +import * as registryModule from "../../state/registry"; + +const PRESETS_DIR = path.join(ROOT, "nemoclaw-blueprint", "policies", "presets"); + +export interface SimulatePolicyOptions { + sandboxName: string; + /** Path to a JSONL trace file, or the literal "-" for stdin. */ + fromFile: string; + /** Candidate policy YAML to test instead of the active sandbox policy. */ + policyFile?: string; + /** Override name applied to candidate presets loaded from policyFile. */ + presetName?: string; + /** Trace lines already read from stdin when fromFile is "-". */ + stdinLines?: string[]; +} + +export type SimulatePolicyResult = + | { kind: "ok"; summary: SimulationSummary; notes: string[] } + | { kind: "error"; lines: string[] }; + +export interface SimulatePolicyDeps { + fileExists?: (p: string) => boolean; + loadTrace?: (p: string) => ParsedTrace; + loadPolicy?: (p: string) => ParsedPreset[]; + getSandboxPolicies?: (name: string) => string[]; + getCustomPolicies?: (name: string) => Array<{ name: string; content: string }>; + presetsDir?: string; +} + +function defaultGetSandboxPolicies(name: string): string[] { + return registryModule.getSandbox(name)?.policies ?? []; +} + +function defaultGetCustomPolicies(name: string): Array<{ name: string; content: string }> { + return registryModule + .getCustomPolicies(name) + .map((entry) => ({ name: entry.name, content: entry.content })); +} + +/** + * Default file-backed loaders. The parsing itself is pure and lives in + * `src/lib/policy/simulate`; only the reads happen here at the host + * boundary. + */ +function defaultLoadTrace(filePath: string): ParsedTrace { + return parseTraceLines(fs.readFileSync(filePath, "utf8").split("\n")); +} + +function defaultLoadPolicy(filePath: string): ParsedPreset[] { + return parsePolicyContent(fs.readFileSync(filePath, "utf8")); +} + +/** + * Load the sandbox's registered policy set: built-in preset files named by + * the registry plus custom/generated policy content recorded on the sandbox + * entry (which includes generated MCP bridge policies). Sources that cannot + * be loaded are reported in `notes` — never silently skipped. + */ +function loadRegisteredPolicySet( + sandboxName: string, + deps: Required< + Pick< + SimulatePolicyDeps, + "fileExists" | "loadPolicy" | "getSandboxPolicies" | "getCustomPolicies" | "presetsDir" + > + >, +): { presets: ParsedPreset[]; notes: string[] } { + const notes: string[] = []; + const presets: ParsedPreset[] = []; + + for (const presetName of deps.getSandboxPolicies(sandboxName)) { + const presetFile = path.join(deps.presetsDir, `${presetName}.yaml`); + if (!deps.fileExists(presetFile)) { + notes.push( + `Applied preset '${presetName}' has no readable file at ${presetFile}; its allows are not evaluated (results may under-report allowed).`, + ); + continue; + } + try { + presets.push(...deps.loadPolicy(presetFile)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + notes.push( + `Applied preset '${presetName}' could not be parsed (${message}); its allows are not evaluated (results may under-report allowed).`, + ); + } + } + + for (const custom of deps.getCustomPolicies(sandboxName)) { + try { + const parsed = parsePolicyContent(custom.content); + if (parsed.length === 0) { + notes.push( + `Custom policy '${custom.name}' contains no evaluable endpoints; it is not part of this simulation.`, + ); + continue; + } + presets.push(...parsed.map((p) => ({ ...p, name: `${custom.name}` }))); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + notes.push( + `Custom policy '${custom.name}' could not be parsed (${message}); its allows are not evaluated (results may under-report allowed).`, + ); + } + } + + return { presets, notes }; +} + +export function simulateSandboxPolicy( + options: SimulatePolicyOptions, + deps: SimulatePolicyDeps = {}, +): SimulatePolicyResult { + const fileExists = deps.fileExists ?? fs.existsSync; + const loadTrace = deps.loadTrace ?? defaultLoadTrace; + const loadPolicy = deps.loadPolicy ?? defaultLoadPolicy; + const getSandboxPolicies = deps.getSandboxPolicies ?? defaultGetSandboxPolicies; + const getCustomPolicies = deps.getCustomPolicies ?? defaultGetCustomPolicies; + const presetsDir = deps.presetsDir ?? PRESETS_DIR; + + let trace: ParsedTrace; + if (options.fromFile === "-") { + trace = parseTraceLines(options.stdinLines ?? []); + } else { + if (!fileExists(options.fromFile)) { + return { kind: "error", lines: [`Trace file not found: ${options.fromFile}`] }; + } + try { + trace = loadTrace(options.fromFile); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + kind: "error", + lines: [`Failed to read trace file ${options.fromFile}: ${message}`], + }; + } + } + + if (trace.requests.length === 0 && trace.invalidLines.length === 0) { + return { + kind: "error", + lines: [ + 'No trace requests found. Check that the file is JSONL with a "host" field per line.', + ], + }; + } + + let presets: ParsedPreset[]; + const notes: string[] = []; + if (options.policyFile) { + if (!fileExists(options.policyFile)) { + return { kind: "error", lines: [`Policy file not found: ${options.policyFile}`] }; + } + try { + presets = loadPolicy(options.policyFile); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + kind: "error", + lines: [`Failed to parse policy file ${options.policyFile}: ${message}`], + }; + } + if (presets.length === 0) { + return { + kind: "error", + lines: [`No parseable endpoints found in policy file: ${options.policyFile}`], + }; + } + if (options.presetName) { + const name = options.presetName; + presets = presets.map((p) => ({ ...p, name })); + } + notes.push( + "Candidate mode: only the provided policy file was evaluated; the sandbox's registered policy set was not.", + ); + } else { + const registered = loadRegisteredPolicySet(options.sandboxName, { + fileExists, + loadPolicy, + getSandboxPolicies, + getCustomPolicies, + presetsDir, + }); + presets = registered.presets; + notes.push(...registered.notes); + if (presets.length === 0) { + return { + kind: "error", + lines: [ + `No registered policy content found for sandbox "${options.sandboxName}".`, + `Add a preset first: nemoclaw ${options.sandboxName} policy-add `, + ...registered.notes, + ], + }; + } + } + + return { kind: "ok", summary: simulate(trace.requests, presets, trace.invalidLines), notes }; +} diff --git a/src/lib/cli/public-display-defaults.ts b/src/lib/cli/public-display-defaults.ts index 950a3d6266e..3a947f6b505 100644 --- a/src/lib/cli/public-display-defaults.ts +++ b/src/lib/cli/public-display-defaults.ts @@ -386,6 +386,14 @@ const PUBLIC_DISPLAY_LAYOUT: Record = { flags: "(--yes, -y, --dry-run)", }, ], + "sandbox:policy:simulate": [ + { + group: "Policy Presets", + order: 21, + description: "Dry-run the active or a candidate policy against a recorded trace", + flags: "--from-file [--policy-file ] [--json]", + }, + ], "sandbox:rebuild": [ { group: "Sandbox Management", diff --git a/src/lib/cli/public-route-metadata.ts b/src/lib/cli/public-route-metadata.ts index 917d7c8d271..bc05f8a94a0 100644 --- a/src/lib/cli/public-route-metadata.ts +++ b/src/lib/cli/public-route-metadata.ts @@ -30,6 +30,7 @@ export const SANDBOX_ROUTE_OVERRIDES: Record = { "sandbox:policy:get": ["policy-get"], "sandbox:policy:list": ["policy-list"], "sandbox:policy:remove": ["policy-remove"], + "sandbox:policy:simulate": ["policy-simulate"], }; function commandIdTokens(commandId: string): string[] { diff --git a/src/lib/policy/simulate.test.ts b/src/lib/policy/simulate.test.ts new file mode 100644 index 00000000000..0bdd39804a0 --- /dev/null +++ b/src/lib/policy/simulate.test.ts @@ -0,0 +1,542 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + parsePolicyContent, + parseTraceLines, + renderSimulationReport, + simulate, + summaryNeedsAttention, +} from "./simulate"; + +const SLACK_PRESET = { + name: "slack", + endpoints: [ + { + host: "slack.com", + port: 443, + enforcement: "enforce", + rules: [ + { allow: { method: "GET", path: "/**" } }, + { allow: { method: "POST", path: "/**" } }, + ], + }, + { + host: "*.slack.com", + port: 443, + enforcement: "enforce", + rules: [ + { allow: { method: "GET", path: "/**" } }, + { allow: { method: "POST", path: "/**" } }, + ], + }, + ], +}; + +const GITHUB_PRESET = { + name: "github", + endpoints: [ + { + host: "api.github.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "GET", path: "/**" } }], + }, + { + host: "raw.githubusercontent.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "GET", path: "/**" } }], + }, + ], +}; + +describe("parseTraceLines", () => { + it("parses valid JSONL lines", () => { + const lines = [ + '{"host":"api.slack.com","port":443,"method":"POST","path":"/api/chat.postMessage"}', + '{"host":"api.github.com","port":443}', + ]; + const trace = parseTraceLines(lines); + expect(trace.requests).toHaveLength(2); + expect(trace.invalidLines).toHaveLength(0); + expect(trace.requests[0]).toMatchObject({ host: "api.slack.com", port: 443 }); + expect(trace.requests[1]).toMatchObject({ host: "api.github.com" }); + }); + + it("skips blank lines and comment lines without reporting them invalid", () => { + const lines = ["", "# comment", '{"host":"api.slack.com"}']; + const trace = parseTraceLines(lines); + expect(trace.requests).toHaveLength(1); + expect(trace.invalidLines).toHaveLength(0); + }); + + it("reports lines without a host field as invalid with their line number", () => { + const lines = ['{"host":"ok.example.com"}', '{"port":443,"method":"GET"}']; + const trace = parseTraceLines(lines); + expect(trace.requests).toHaveLength(1); + expect(trace.invalidLines).toHaveLength(1); + expect(trace.invalidLines[0].line).toBe(2); + expect(trace.invalidLines[0].reason).toContain("host"); + }); + + it("reports invalid JSON as invalid instead of silently dropping it", () => { + const lines = ["not-json", '{"host":"api.slack.com"}']; + const trace = parseTraceLines(lines); + expect(trace.requests).toHaveLength(1); + expect(trace.invalidLines).toHaveLength(1); + expect(trace.invalidLines[0].line).toBe(1); + expect(trace.invalidLines[0].reason).toBe("not valid JSON"); + }); + + it("reports non-object JSON rows as invalid", () => { + const trace = parseTraceLines(['["host","api.slack.com"]', '"just-a-string"']); + expect(trace.requests).toHaveLength(0); + expect(trace.invalidLines).toHaveLength(2); + }); +}); + +describe("simulate", () => { + it("allows a request matching an active preset", () => { + const req = { host: "api.slack.com", port: 443, method: "POST", path: "/api/chat.postMessage" }; + const summary = simulate([req], [SLACK_PRESET]); + expect(summary.results[0].verdict).toBe("allowed"); + expect(summary.results[0].allowedBy).toBe("slack"); + }); + + it("marks a request as uncovered when no preset matches the host", () => { + const req = { host: "api.openai.com", port: 443, method: "POST" }; + const summary = simulate([req], [SLACK_PRESET]); + expect(summary.results[0].verdict).toBe("uncovered"); + }); + + it("allows requests to wildcard subdomains", () => { + const req = { host: "files.slack.com", port: 443, method: "GET", path: "/files/foo" }; + const summary = simulate([req], [SLACK_PRESET]); + expect(summary.results[0].verdict).toBe("allowed"); + }); + + it("does not let a single-label host wildcard cross DNS labels", () => { + const req = { host: "a.b.slack.com", port: 443, method: "GET", path: "/x" }; + const summary = simulate([req], [SLACK_PRESET]); + expect(summary.results[0].verdict).toBe("uncovered"); + }); + + it("lets a double-star host wildcard cross DNS labels", () => { + const deepPreset = { + name: "deep", + endpoints: [ + { + host: "**.slack.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "GET", path: "/**" } }], + }, + ], + }; + const req = { host: "a.b.slack.com", port: 443, method: "GET", path: "/x" }; + const summary = simulate([req], [deepPreset]); + expect(summary.results[0].verdict).toBe("allowed"); + }); + + it("reports blocked when the endpoint covers the host but denies the method", () => { + const presetWithGetOnly = { + name: "restricted", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "GET", path: "/**" } }], + }, + ], + }; + const postReq = { host: "api.example.com", port: 443, method: "POST", path: "/x" }; + const getReq = { host: "api.example.com", port: 443, method: "GET", path: "/x" }; + const postSummary = simulate([postReq], [presetWithGetOnly]); + const getSummary = simulate([getReq], [presetWithGetOnly]); + expect(postSummary.results[0].verdict).toBe("blocked"); + expect(postSummary.blocked).toBe(1); + expect(getSummary.results[0].verdict).toBe("allowed"); + }); + + it("prefers allowed over blocked when a later preset permits the request", () => { + const denyPreset = { + name: "deny-get-only", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "GET", path: "/**" } }], + }, + ], + }; + const allowPreset = { + name: "allow-post", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "POST", path: "/**" } }], + }, + ], + }; + const req = { host: "api.example.com", port: 443, method: "POST", path: "/x" }; + const summary = simulate([req], [denyPreset, allowPreset]); + expect(summary.results[0].verdict).toBe("allowed"); + expect(summary.results[0].allowedBy).toBe("allow-post"); + }); + + it("counts multiple verdicts correctly", () => { + const requests = [ + { host: "api.slack.com", port: 443, method: "POST", path: "/x" }, + { host: "api.github.com", port: 443, method: "GET", path: "/y" }, + { host: "evil.example.com", port: 80, method: "GET", path: "/z" }, + ]; + const summary = simulate(requests, [SLACK_PRESET, GITHUB_PRESET]); + expect(summary.allowed).toBe(2); + expect(summary.uncovered).toBe(1); + expect(summary.blocked).toBe(0); + expect(summary.unknown).toBe(0); + }); + + it("uses first matching preset when multiple could match", () => { + const req = { host: "api.slack.com", port: 443, method: "GET", path: "/x" }; + const summary = simulate([req], [SLACK_PRESET, GITHUB_PRESET]); + expect(summary.results[0].allowedBy).toBe("slack"); + }); + + it("allows monitor-mode endpoints regardless of rules", () => { + const monitorPreset = { + name: "monitor", + endpoints: [{ host: "api.example.com", port: 443, enforcement: "monitor" }], + }; + const req = { host: "api.example.com", port: 443, method: "DELETE", path: "/x" }; + const summary = simulate([req], [monitorPreset]); + expect(summary.results[0].verdict).toBe("allowed"); + expect(summary.results[0].matchedRule).toContain("monitor"); + }); +}); + +describe("simulate fail-closed semantics", () => { + it("reports unknown when the trace row lacks a method the rule constrains", () => { + const req = { host: "api.slack.com", port: 443, path: "/x" }; + const summary = simulate([req], [SLACK_PRESET]); + expect(summary.results[0].verdict).toBe("unknown"); + expect(summary.results[0].reason).toContain("method/path"); + expect(summary.unknown).toBe(1); + }); + + it("reports unknown when the trace row lacks a path the rule constrains", () => { + const pathScoped = { + name: "path-scoped", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "GET", path: "/api/*" } }], + }, + ], + }; + const req = { host: "api.example.com", port: 443, method: "GET" }; + const summary = simulate([req], [pathScoped]); + expect(summary.results[0].verdict).toBe("unknown"); + }); + + it("still allows when the rule is fully wildcarded even if the row lacks fields", () => { + const wildcard = { + name: "wildcard", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "*", path: "/**" } }], + }, + ], + }; + const req = { host: "api.example.com", port: 443 }; + const summary = simulate([req], [wildcard]); + expect(summary.results[0].verdict).toBe("allowed"); + }); + + it("reports unknown when the endpoint pins a port the trace row omits", () => { + const req = { host: "api.slack.com", method: "GET", path: "/x" }; + const summary = simulate([req], [SLACK_PRESET]); + expect(summary.results[0].verdict).toBe("unknown"); + expect(summary.results[0].reason).toContain("port"); + }); + + it("reports unknown instead of allowed for endpoints with unevaluated constraints", () => { + const constrained = { + name: "constrained", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "GET", path: "/**" } }], + unevaluatedConstraints: ["allowed_ips", "protocol", "tls"], + }, + ], + }; + const req = { host: "api.example.com", port: 443, method: "GET", path: "/x" }; + const summary = simulate([req], [constrained]); + expect(summary.results[0].verdict).toBe("unknown"); + expect(summary.results[0].reason).toContain("allowed_ips"); + expect(summary.results[0].reason).toContain("protocol"); + expect(summary.results[0].reason).toContain("tls"); + }); + + it("reports unknown for monitor endpoints that carry unevaluated constraints", () => { + const monitorConstrained = { + name: "monitor-constrained", + endpoints: [ + { + host: "api.example.com", + enforcement: "monitor", + unevaluatedConstraints: ["ancestry"], + }, + ], + }; + const req = { host: "api.example.com", method: "GET", path: "/x" }; + const summary = simulate([req], [monitorConstrained]); + expect(summary.results[0].verdict).toBe("unknown"); + expect(summary.results[0].reason).toContain("ancestry"); + }); + + it("lets unknown outrank a firm block from another endpoint", () => { + const blocking = { + name: "blocking", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "GET", path: "/**" } }], + }, + ], + }; + const maybeAllowing = { + name: "maybe-allowing", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "POST", path: "/**" } }], + unevaluatedConstraints: ["mcp"], + }, + ], + }; + const req = { host: "api.example.com", port: 443, method: "POST", path: "/x" }; + const summary = simulate([req], [blocking, maybeAllowing]); + expect(summary.results[0].verdict).toBe("unknown"); + }); + + it("lets a proven allow outrank unknown from another endpoint", () => { + const maybe = { + name: "maybe", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "POST", path: "/**" } }], + unevaluatedConstraints: ["tls"], + }, + ], + }; + const proven = { + name: "proven", + endpoints: [ + { + host: "api.example.com", + port: 443, + enforcement: "enforce", + rules: [{ allow: { method: "POST", path: "/**" } }], + }, + ], + }; + const req = { host: "api.example.com", port: 443, method: "POST", path: "/x" }; + const summary = simulate([req], [maybe, proven]); + expect(summary.results[0].verdict).toBe("allowed"); + expect(summary.results[0].allowedBy).toBe("proven"); + }); + + it("carries invalid trace lines into the summary", () => { + const trace = parseTraceLines(["oops", '{"host":"api.slack.com","port":443}']); + const summary = simulate(trace.requests, [SLACK_PRESET], trace.invalidLines); + expect(summary.invalidTraceLines).toHaveLength(1); + expect(summaryNeedsAttention(summary)).toBe(true); + }); +}); + +describe("summaryNeedsAttention", () => { + it("is false only when every request is a proven allow and no rows were invalid", () => { + const req = { host: "api.slack.com", port: 443, method: "POST", path: "/x" }; + const clean = simulate([req], [SLACK_PRESET]); + expect(summaryNeedsAttention(clean)).toBe(false); + }); + + it("is true for blocked, uncovered, and unknown requests", () => { + const blocked = simulate( + [{ host: "api.github.com", port: 443, method: "POST", path: "/x" }], + [GITHUB_PRESET], + ); + const uncovered = simulate( + [{ host: "nowhere.example.com", port: 443, method: "GET", path: "/x" }], + [GITHUB_PRESET], + ); + const unknown = simulate([{ host: "api.github.com", port: 443 }], [GITHUB_PRESET]); + expect(summaryNeedsAttention(blocked)).toBe(true); + expect(summaryNeedsAttention(uncovered)).toBe(true); + expect(summaryNeedsAttention(unknown)).toBe(true); + }); +}); + +describe("parsePolicyContent", () => { + it("parses preset endpoints from policy YAML", () => { + const yaml = [ + "network_policies:", + " slack:", + " name: slack", + " endpoints:", + " - host: api.slack.com", + " port: 443", + " rules:", + " - allow: { method: POST, path: '/**' }", + ].join("\n"); + const presets = parsePolicyContent(yaml); + expect(presets).toHaveLength(1); + expect(presets[0].endpoints[0].host).toBe("api.slack.com"); + expect(presets[0].endpoints[0].unevaluatedConstraints).toBeUndefined(); + }); + + it("throws a descriptive error on invalid YAML", () => { + expect(() => parsePolicyContent("a: [unclosed")).toThrow(/Invalid policy YAML/); + }); + + it("returns empty for YAML without network_policies", () => { + expect(parsePolicyContent("preset:\n name: x")).toHaveLength(0); + }); + + it("drops endpoints whose rules are mapping-shaped instead of a list", () => { + const yaml = [ + "network_policies:", + " broken:", + " endpoints:", + " - host: api.example.com", + " rules:", + " allow: { method: GET }", + " - host: ok.example.com", + " rules:", + " - allow: { method: GET, path: '/**' }", + ].join("\n"); + const presets = parsePolicyContent(yaml); + expect(presets).toHaveLength(1); + expect(presets[0].endpoints).toHaveLength(1); + expect(presets[0].endpoints[0].host).toBe("ok.example.com"); + }); + + it("records unevaluated endpoint constraints such as protocol, allowed_ips, and tls", () => { + const yaml = [ + "network_policies:", + " hardened:", + " endpoints:", + " - host: api.example.com", + " port: 443", + " protocol: https", + " allowed_ips: ['203.0.113.7']", + " tls: { min_version: '1.3' }", + " rules:", + " - allow: { method: GET, path: '/**' }", + ].join("\n"); + const presets = parsePolicyContent(yaml); + const constraints = presets[0].endpoints[0].unevaluatedConstraints ?? []; + expect(constraints).toContain("protocol"); + expect(constraints).toContain("allowed_ips"); + expect(constraints).toContain("tls"); + const req = { host: "api.example.com", port: 443, method: "GET", path: "/x" }; + const summary = simulate([req], presets); + expect(summary.results[0].verdict).toBe("unknown"); + }); + + it("records unevaluated rule keys such as deny and ancestry", () => { + const yaml = [ + "network_policies:", + " denying:", + " endpoints:", + " - host: api.example.com", + " port: 443", + " rules:", + " - allow: { method: GET, path: '/**' }", + " ancestry: sandbox-only", + " - deny: { method: POST }", + ].join("\n"); + const presets = parsePolicyContent(yaml); + const constraints = presets[0].endpoints[0].unevaluatedConstraints ?? []; + expect(constraints).toContain("rules.ancestry"); + expect(constraints).toContain("rules.deny"); + const req = { host: "api.example.com", port: 443, method: "GET", path: "/x" }; + const summary = simulate([req], presets); + expect(summary.results[0].verdict).toBe("unknown"); + }); + + it("records unevaluated allow keys such as allow.protocol", () => { + const yaml = [ + "network_policies:", + " scoped:", + " endpoints:", + " - host: api.example.com", + " port: 443", + " rules:", + " - allow: { method: GET, path: '/**', protocol: https }", + ].join("\n"); + const presets = parsePolicyContent(yaml); + const constraints = presets[0].endpoints[0].unevaluatedConstraints ?? []; + expect(constraints).toContain("rules.allow.protocol"); + }); +}); + +describe("renderSimulationReport", () => { + it("renders JSON when json=true", () => { + const summary = simulate( + [{ host: "api.slack.com", port: 443, method: "GET", path: "/x" }], + [SLACK_PRESET], + ); + const report = renderSimulationReport(summary, true); + const parsed = JSON.parse(report) as { totalRequests: number; unknown: number }; + expect(parsed.totalRequests).toBe(1); + expect(parsed.unknown).toBe(0); + }); + + it("renders human-readable report with the static-evaluation disclaimer", () => { + const requests = [ + { host: "api.slack.com", port: 443, method: "POST", path: "/x" }, + { host: "unknown.example.com", port: 443, method: "GET", path: "/x" }, + ]; + const summary = simulate(requests, [SLACK_PRESET]); + const report = renderSimulationReport(summary, false); + expect(report).toContain("ALLOWED"); + expect(report).toContain("UNCOVERED"); + expect(report).toContain("slack"); + expect(report).toContain("static evaluation"); + expect(report).toContain("drift is not detected"); + }); + + it("lists invalid trace rows and unknown verdicts with reasons", () => { + const trace = parseTraceLines(["broken-row", '{"host":"api.slack.com","port":443}']); + const summary = simulate(trace.requests, [SLACK_PRESET], trace.invalidLines); + const report = renderSimulationReport(summary, false); + expect(report).toContain("INVALID TRACE ROWS"); + expect(report).toContain("line 1: not valid JSON"); + expect(report).toContain("UNKNOWN"); + expect(report).toContain("method/path"); + }); +}); diff --git a/src/lib/policy/simulate.ts b/src/lib/policy/simulate.ts new file mode 100644 index 00000000000..32f11180fe6 --- /dev/null +++ b/src/lib/policy/simulate.ts @@ -0,0 +1,541 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Policy simulation engine. + * + * Statically evaluates a list of observed network requests against parsed + * policy content. The engine is deliberately fail-closed: it claims a + * request "matches an allow rule" only when every evaluated dimension is + * proven from the trace row, and reports `unknown` whenever the policy + * carries constraints this engine does not evaluate (protocol, allowed_ips, + * TLS, ancestry, MCP, deny rules, …) or the trace row lacks a field a rule + * constrains. It never consults live gateway state, so registry/gateway + * drift is out of scope and callers must present results as static. + * + * Trace file format — one JSON object per line (JSONL): + * {"host":"api.slack.com","port":443,"method":"POST","path":"/api/chat.postMessage"} + * + * `host` is required per row; rows that are not valid JSON or lack a string + * `host` are reported as invalid, not silently dropped. + */ + +import YAML from "yaml"; + +import type { PolicyObject, PolicyValue } from "./preset-parsing"; +import { isPolicyDocument, isPolicyObject } from "./preset-parsing"; + +export type SimulateVerdict = "allowed" | "blocked" | "uncovered" | "unknown"; + +export interface TraceRequest { + host: string; + port?: number; + method?: string; + path?: string; + /** Optional label for display (e.g., which agent or command produced it) */ + label?: string; +} + +export interface InvalidTraceLine { + /** 1-based line number within the provided input. */ + line: number; + reason: string; + /** Truncated raw content for diagnosis. */ + excerpt: string; +} + +export interface ParsedTrace { + requests: TraceRequest[]; + invalidLines: InvalidTraceLine[]; +} + +export interface SimulateResult { + request: TraceRequest; + verdict: SimulateVerdict; + /** Preset name whose allow rule was proven, when verdict is "allowed" */ + allowedBy?: string; + /** Rule description, when verdict is "allowed" */ + matchedRule?: string; + /** Why the verdict is "unknown", when it is */ + reason?: string; +} + +export interface SimulationSummary { + totalRequests: number; + allowed: number; + blocked: number; + uncovered: number; + unknown: number; + invalidTraceLines: InvalidTraceLine[]; + results: SimulateResult[]; +} + +export interface PolicyEndpoint { + host: string; + port?: number | string; + enforcement?: string; + rules?: Array<{ allow?: { method?: string; path?: string } }>; + /** Endpoint keys the engine recognized but does not evaluate. */ + unevaluatedConstraints?: string[]; +} + +export interface ParsedPreset { + name: string; + endpoints: PolicyEndpoint[]; +} + +/** + * Endpoint keys this engine evaluates. Anything else on an endpoint (for + * example `protocol`, `allowed_ips`, `tls`, `ancestry`, `mcp`) is a + * constraint the engine cannot prove, so a would-be allow through that + * endpoint degrades to `unknown` instead of over-claiming. + */ +const EVALUATED_ENDPOINT_KEYS = new Set(["host", "port", "enforcement", "rules"]); +const EVALUATED_RULE_KEYS = new Set(["allow"]); +const EVALUATED_ALLOW_KEYS = new Set(["method", "path"]); + +/** + * Match a glob pattern against a path or method string. + * Supports `*` (any single path segment) and `**` (any number of segments). + */ +function globMatch(pattern: string, value: string): boolean { + if (pattern === "**" || pattern === "*") return true; + // Escape regex special chars except * which becomes .* or [^/]* + const escaped = pattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*\*/g, "<<>>") + .replace(/\*/g, "[^/]*") + .replace(/<<>>/g, ".*"); + return new RegExp(`^${escaped}$`, "i").test(value); +} + +/** + * Match a hostname pattern against a host. For hostnames, `*` matches a + * single DNS label (dots are separators), so `*.example.com` matches + * `api.example.com` but not `a.b.example.com`. `**` crosses labels. + */ +function hostMatches(pattern: string, host: string): boolean { + if (pattern === "**" || pattern === "*") return true; + const escaped = pattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*\*/g, "<<>>") + .replace(/\*/g, "[^.]*") + .replace(/<<>>/g, ".*"); + return new RegExp(`^${escaped}$`, "i").test(host); +} + +type TriState = "match" | "no-match" | "unprovable"; + +/** + * Fail-closed port comparison: an endpoint pinned to a specific port cannot + * be proven to cover a trace row that omits the port. + */ +function portMatch( + endpointPort: number | string | undefined, + requestPort: number | undefined, +): TriState { + if (endpointPort === undefined || endpointPort === "*") return "match"; + if (requestPort === undefined) return "unprovable"; + return Number(endpointPort) === requestPort ? "match" : "no-match"; +} + +/** + * Fail-closed rule comparison: a rule that constrains method or path cannot + * be proven to match a trace row that omits that field. + */ +function ruleMatch( + rule: { allow?: { method?: string; path?: string } }, + method: string | undefined, + reqPath: string | undefined, +): TriState { + if (!rule.allow) return "no-match"; + const { method: ruleMethod, path: rulePath } = rule.allow; + if (ruleMethod && ruleMethod !== "*" && ruleMethod !== "**") { + if (method === undefined) return "unprovable"; + if (!globMatch(ruleMethod, method)) return "no-match"; + } + if (rulePath && rulePath !== "/**" && rulePath !== "**") { + if (reqPath === undefined) return "unprovable"; + if (!globMatch(rulePath, reqPath)) return "no-match"; + } + return "match"; +} + +type EndpointDecision = + | { verdict: "allowed"; rule: string } + | { verdict: "blocked" } + | { verdict: "unknown"; reason: string } + | null; + +/** + * Evaluate one endpoint against a request. Returns: + * - `{verdict: "allowed", rule}` when every evaluated dimension proves the + * endpoint covers and permits the request, + * - `{verdict: "blocked"}` when the endpoint provably covers the host/port + * and every allow rule provably fails to match, + * - `{verdict: "unknown", reason}` when coverage or a rule match cannot be + * proven (missing trace fields) or the endpoint/rule carries constraints + * this engine does not evaluate, + * - `null` when the endpoint provably does not cover this host/port. + */ +function endpointDecision(endpoint: PolicyEndpoint, req: TraceRequest): EndpointDecision { + if (!hostMatches(endpoint.host, req.host)) return null; + const port = portMatch(endpoint.port, req.port); + if (port === "no-match") return null; + if (port === "unprovable") { + return { + verdict: "unknown", + reason: `endpoint '${endpoint.host}' pins port ${endpoint.port} but the trace row has no port`, + }; + } + const unevaluated = endpoint.unevaluatedConstraints ?? []; + if (endpoint.enforcement === "monitor") { + if (unevaluated.length > 0) { + return { + verdict: "unknown", + reason: `monitor endpoint '${endpoint.host}' carries unevaluated constraints: ${unevaluated.join(", ")}`, + }; + } + return { verdict: "allowed", rule: "monitor (allowed but observed)" }; + } + if (!endpoint.rules || endpoint.rules.length === 0) { + if (unevaluated.length > 0) { + return { + verdict: "unknown", + reason: `endpoint '${endpoint.host}' carries unevaluated constraints: ${unevaluated.join(", ")}`, + }; + } + return { verdict: "allowed", rule: "default allow (no rules)" }; + } + let sawUnprovable = false; + for (const rule of endpoint.rules) { + const match = ruleMatch(rule, req.method, req.path); + if (match === "unprovable") { + sawUnprovable = true; + continue; + } + if (match === "match") { + if (unevaluated.length > 0) { + return { + verdict: "unknown", + reason: `endpoint '${endpoint.host}' carries unevaluated constraints: ${unevaluated.join(", ")}`, + }; + } + const m = rule.allow?.method ?? "*"; + const p = rule.allow?.path ?? "/**"; + return { verdict: "allowed", rule: `allow ${m} ${p}` }; + } + } + if (sawUnprovable) { + return { + verdict: "unknown", + reason: `trace row lacks the method/path needed to evaluate rules on endpoint '${endpoint.host}'`, + }; + } + return { verdict: "blocked" }; +} + +/** + * Accept an endpoint only when it has a string `host` and its `rules` + * field, when present, is array-shaped. Mapping-shaped `rules` from + * malformed YAML would otherwise crash the `for...of` in + * {@link endpointDecision}. Endpoint and rule keys the engine does not + * evaluate are recorded so allow verdicts through them degrade to + * `unknown`. + */ +function toEvaluatedEndpoint(ep: PolicyValue): PolicyEndpoint | null { + if (!isPolicyObject(ep)) return null; + if (typeof ep["host"] !== "string") return null; + const rules = ep["rules"]; + if (rules !== undefined && rules !== null && !Array.isArray(rules)) return null; + + const unevaluated = new Set(); + for (const key of Object.keys(ep)) { + if (!EVALUATED_ENDPOINT_KEYS.has(key)) unevaluated.add(key); + } + if (Array.isArray(rules)) { + for (const rule of rules) { + if (!isPolicyObject(rule)) { + unevaluated.add("rules (non-mapping rule entry)"); + continue; + } + for (const key of Object.keys(rule)) { + if (!EVALUATED_RULE_KEYS.has(key)) unevaluated.add(`rules.${key}`); + } + const allow = rule["allow"]; + if (allow !== undefined && !isPolicyObject(allow)) { + unevaluated.add("rules.allow (non-mapping)"); + continue; + } + if (isPolicyObject(allow)) { + for (const key of Object.keys(allow)) { + if (!EVALUATED_ALLOW_KEYS.has(key)) unevaluated.add(`rules.allow.${key}`); + } + } + } + } + + const result = ep as PolicyObject & PolicyEndpoint; + if (unevaluated.size > 0) { + return { ...result, unevaluatedConstraints: [...unevaluated].sort() }; + } + return result; +} + +function extractEndpoints(policyMap: PolicyObject): ParsedPreset[] { + const presets: ParsedPreset[] = []; + for (const [presetName, presetVal] of Object.entries(policyMap)) { + if (!isPolicyObject(presetVal)) continue; + const networkPolicies = presetVal["network_policies"]; + const topLevelEndpoints = presetVal["endpoints"]; + + // Support both preset-level endpoints and nested network_policies + const policyBlock: PolicyObject = isPolicyObject(networkPolicies) + ? networkPolicies + : isPolicyObject(presetVal) + ? presetVal + : {}; + + const endpoints: PolicyEndpoint[] = []; + + if (Array.isArray(topLevelEndpoints)) { + for (const ep of topLevelEndpoints) { + const evaluated = toEvaluatedEndpoint(ep); + if (evaluated) endpoints.push(evaluated); + } + } + + for (const [, policyVal] of Object.entries(policyBlock)) { + if (!isPolicyObject(policyVal)) continue; + const epList = policyVal["endpoints"]; + if (!Array.isArray(epList)) continue; + for (const ep of epList) { + const evaluated = toEvaluatedEndpoint(ep); + if (evaluated) endpoints.push(evaluated); + } + } + + if (endpoints.length > 0) presets.push({ name: presetName, endpoints }); + } + return presets; +} + +/** + * Parse policy YAML content into a flat list of endpoint presets. + * Pure: takes the file content, not a path. Throws a descriptive Error + * when the content is not valid YAML. + */ +export function parsePolicyContent(content: string): ParsedPreset[] { + let parsed: PolicyValue; + try { + parsed = YAML.parse(content) as PolicyValue; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid policy YAML: ${message}`); + } + if (!isPolicyDocument(parsed)) return []; + + const networkPolicies = parsed["network_policies"]; + if (!isPolicyObject(networkPolicies)) return []; + + return extractEndpoints(networkPolicies); +} + +const INVALID_LINE_EXCERPT_LENGTH = 80; + +/** + * Parse JSONL trace lines. Blank lines and `#` comments are skipped; + * anything else that is not a JSON object with a string `host` is reported + * in `invalidLines` so malformed input can never silently shrink the trace. + */ +export function parseTraceLines(lines: string[]): ParsedTrace { + const requests: TraceRequest[] = []; + const invalidLines: InvalidTraceLine[] = []; + for (const [index, line] of lines.entries()) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const excerpt = + trimmed.length > INVALID_LINE_EXCERPT_LENGTH + ? `${trimmed.slice(0, INVALID_LINE_EXCERPT_LENGTH)}…` + : trimmed; + let obj: Record; + try { + const parsed: unknown = JSON.parse(trimmed); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + invalidLines.push({ line: index + 1, reason: "not a JSON object", excerpt }); + continue; + } + obj = parsed as Record; + } catch { + invalidLines.push({ line: index + 1, reason: "not valid JSON", excerpt }); + continue; + } + if (typeof obj.host !== "string" || obj.host.length === 0) { + invalidLines.push({ line: index + 1, reason: 'missing string "host" field', excerpt }); + continue; + } + requests.push({ + host: obj.host, + port: typeof obj.port === "number" ? obj.port : undefined, + method: typeof obj.method === "string" ? obj.method : undefined, + path: typeof obj.path === "string" ? obj.path : undefined, + label: typeof obj.label === "string" ? obj.label : undefined, + }); + } + return { requests, invalidLines }; +} + +/** + * Evaluate a list of trace requests against the given presets. + * + * Per-request precedence is fail-closed: a proven allow wins; otherwise any + * unprovable evaluation makes the request `unknown` (a firm block elsewhere + * cannot outrank a possible allow); otherwise a proven covered-and-denied + * endpoint yields `blocked`; otherwise `uncovered`. + */ +export function simulate( + requests: TraceRequest[], + presets: ParsedPreset[], + invalidTraceLines: InvalidTraceLine[] = [], +): SimulationSummary { + const results: SimulateResult[] = []; + + for (const req of requests) { + let verdict: SimulateVerdict = "uncovered"; + let allowedBy: string | undefined; + let matchedRule: string | undefined; + let reason: string | undefined; + + for (const preset of presets) { + for (const endpoint of preset.endpoints) { + const decision = endpointDecision(endpoint, req); + if (decision === null) continue; + if (decision.verdict === "allowed") { + verdict = "allowed"; + allowedBy = preset.name; + matchedRule = decision.rule; + reason = undefined; + break; + } + if (decision.verdict === "unknown") { + verdict = "unknown"; + reason = reason ?? `${preset.name}: ${decision.reason}`; + continue; + } + if (verdict === "uncovered") verdict = "blocked"; + } + if (verdict === "allowed") break; + } + + results.push({ request: req, verdict, allowedBy, matchedRule, reason }); + } + + const count = (v: SimulateVerdict) => results.filter((r) => r.verdict === v).length; + + return { + totalRequests: results.length, + allowed: count("allowed"), + blocked: count("blocked"), + uncovered: count("uncovered"), + unknown: count("unknown"), + invalidTraceLines, + results, + }; +} + +/** + * True when the summary contains anything an operator must act on before + * trusting the trace as fully covered: blocked, uncovered, or unknown + * requests, or trace rows that could not be parsed. + */ +export function summaryNeedsAttention(summary: SimulationSummary): boolean { + return ( + summary.blocked > 0 || + summary.uncovered > 0 || + summary.unknown > 0 || + summary.invalidTraceLines.length > 0 + ); +} + +/** + * Render a simulation summary as a human-readable report. + */ +export function renderSimulationReport(summary: SimulationSummary, json: boolean): string { + if (json) return JSON.stringify(summary, null, 2); + + const lines: string[] = []; + lines.push(`Policy Simulation (static) — ${summary.totalRequests} trace request(s) evaluated`); + lines.push( + ` Allowed: ${summary.allowed} Blocked: ${summary.blocked} Uncovered: ${summary.uncovered}` + + ` Unknown: ${summary.unknown} Invalid rows: ${summary.invalidTraceLines.length}`, + ); + lines.push(""); + + if (summary.invalidTraceLines.length > 0) { + lines.push("INVALID TRACE ROWS (not evaluated)"); + for (const invalid of summary.invalidTraceLines) { + lines.push(` ! line ${invalid.line}: ${invalid.reason} — ${invalid.excerpt}`); + } + lines.push(""); + } + + const groups: Record = { + allowed: [], + blocked: [], + uncovered: [], + unknown: [], + }; + for (const r of summary.results) groups[r.verdict].push(r); + + if (groups.allowed.length > 0) { + lines.push("ALLOWED (matches an allow rule on every evaluated dimension)"); + for (const r of groups.allowed) { + const req = formatReq(r.request); + lines.push(` ✓ ${req} → ${r.allowedBy} (${r.matchedRule})`); + } + lines.push(""); + } + + if (groups.blocked.length > 0) { + lines.push("BLOCKED (covered by an endpoint whose rules deny it)"); + for (const r of groups.blocked) { + lines.push(` ✗ ${formatReq(r.request)}`); + } + lines.push(""); + } + + if (groups.unknown.length > 0) { + lines.push("UNKNOWN (cannot be proven from the trace and evaluated dimensions)"); + for (const r of groups.unknown) { + lines.push(` ? ${formatReq(r.request)}`); + if (r.reason) lines.push(` ${r.reason}`); + } + lines.push(""); + } + + if (groups.uncovered.length > 0) { + lines.push("UNCOVERED (no evaluated endpoint covers the request)"); + for (const r of groups.uncovered) { + lines.push(` - ${formatReq(r.request)}`); + } + lines.push(""); + } + + lines.push( + "Note: this is a static evaluation of registered policy content over host, port, method, and", + ); + lines.push( + "path only. Constraints the engine does not evaluate are reported as UNKNOWN, and live", + ); + lines.push("gateway state is not consulted, so registry/gateway drift is not detected."); + + return lines.join("\n"); +} + +function formatReq(req: TraceRequest): string { + const port = req.port ? `:${req.port}` : ""; + const method = req.method ? `${req.method} ` : ""; + const reqPath = req.path ?? ""; + const label = req.label ? ` [${req.label}]` : ""; + return `${method}${req.host}${port}${reqPath}${label}`; +} diff --git a/test/package-contract/cli/command-registry.test.ts b/test/package-contract/cli/command-registry.test.ts index 4252f930f1a..7cf1bd1c397 100644 --- a/test/package-contract/cli/command-registry.test.ts +++ b/test/package-contract/cli/command-registry.test.ts @@ -56,16 +56,17 @@ describe("command-registry", () => { }); describe("sandboxCommands()", () => { - it("should return exactly 58 entries", () => { + it("should return exactly 59 entries", () => { // 50 visible + 8 hidden (shields×3 + config get/set/rotate-token + // inference get/set). // 50 visible includes the sessions group (root + list + reset + delete + // export), the agents quartet (add + apply + delete + list), the // singular `agent` passthrough that forwards to `openclaw agent`, and // the download + upload host-side openshell wrappers, plus five MCP - // bridge display entries under the `mcp` parent and the gateway restart - // command under the `gateway` parent. - expect(sandboxCommands()).toHaveLength(58); + // bridge display entries under the `mcp` parent, the gateway restart + // command under the `gateway` parent, and the policy-simulate dry-run + // command under the `policy` parent. + expect(sandboxCommands()).toHaveLength(59); }); it("every entry has scope sandbox", () => { @@ -224,9 +225,9 @@ describe("command-registry", () => { }); describe("sandboxActionTokens()", () => { - it("returns exactly 33 unique action tokens including empty string", () => { + it("returns exactly 34 unique action tokens including empty string", () => { const tokens = sandboxActionTokens(); - expect(tokens).toHaveLength(33); + expect(tokens).toHaveLength(34); // Must contain every first-level sandbox action plus the empty default action. const expected = new Set([ "agent", @@ -244,6 +245,7 @@ describe("command-registry", () => { "policy-get", "policy-remove", "policy-list", + "policy-simulate", "hosts-add", "hosts-list", "hosts-remove", diff --git a/test/package-contract/cli/public-argv-translation.test.ts b/test/package-contract/cli/public-argv-translation.test.ts index 991c39c060f..ce31a18ee0e 100644 --- a/test/package-contract/cli/public-argv-translation.test.ts +++ b/test/package-contract/cli/public-argv-translation.test.ts @@ -93,6 +93,7 @@ describe("public route/display separation", () => { "sandbox:policy:get", "sandbox:policy:list", "sandbox:policy:remove", + "sandbox:policy:simulate", ]); expect(sandboxRouteTokens("sandbox:gateway:token")).toEqual(["gateway-token"]); expect(sandboxRouteTokens("sandbox:config:rotate-token")).toEqual(["config", "rotate-token"]);