diff --git a/docs/network-policy/explain-network-policy-to-agents.mdx b/docs/network-policy/explain-network-policy-to-agents.mdx index 977c8b18472..a225323bae8 100644 --- a/docs/network-policy/explain-network-policy-to-agents.mdx +++ b/docs/network-policy/explain-network-policy-to-agents.mdx @@ -62,6 +62,7 @@ Each active preset includes a `verification` value: | `verified` | The registry lists the preset, and the gateway confirms enforcement. | | `registry-only` | The registry lists the preset, but the gateway does not enforce it. Treat the allowed hosts as unverified. | | `gateway-only` | The gateway enforces a preset that the registry does not list. | +| `agent-base` | The gateway enforces this preset because it belongs to the agent's own base policy (`agents//policy-additions.yaml`), not because the operator applied it. It is active, not drift. `policy add` is unnecessary and would record the preset as operator-applied. | | `gateway-unavailable` | NemoClaw could not probe the gateway. Treat the report as advisory until the gateway is reachable. | ## Classify a Failed Request @@ -81,7 +82,7 @@ The classifier evaluates conditions in this order: Network-block error codes include `EHOSTUNREACH`, `ENETUNREACH`, `ENOTFOUND`, `ECONNREFUSED`, `ETIMEDOUT`, and `EAI_AGAIN`. A block code on a host from a `registry-only` or `gateway-unavailable` preset produces a low-confidence policy verdict. -A block code on a host from a verified preset stays `unknown` because the gateway already confirmed enforcement. +A block code on a host from a `verified`, `gateway-only`, or `agent-base` preset stays `unknown` with high confidence because the gateway confirmed enforcement. Each verdict includes `confidence` set to `high` or `low`. Low confidence means the agent must report multiple possibilities instead of treating one next step as authoritative. diff --git a/src/lib/policy/agent-base-preset.test.ts b/src/lib/policy/agent-base-preset.test.ts new file mode 100644 index 00000000000..f9eae5b385f --- /dev/null +++ b/src/lib/policy/agent-base-preset.test.ts @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { AGENTS_DIR } from "../agent/defs"; +import * as registry from "../state/registry"; +import { isAgentBasePreset } from "./index"; + +const tempAgentDirs: string[] = []; + +function createAgentFixture(policyAdditions?: string): string { + const agentName = `agent-base-preset-${randomUUID()}`; + const agentDir = path.join(AGENTS_DIR, agentName); + tempAgentDirs.push(agentDir); + fs.mkdirSync(agentDir, { recursive: true }); + fs.writeFileSync( + path.join(agentDir, "manifest.yaml"), + `name: ${agentName}\ndisplay_name: Agent Base Preset Fixture\n`, + ); + fs.writeFileSync( + path.join(agentDir, "policy-additions.yaml"), + policyAdditions ?? + `version: 1 +network_policies: + github: + name: github + endpoints: + - host: api.github.com + port: 443 + access: full + binaries: + - path: /usr/bin/git +`, + ); + return agentName; +} + +afterEach(() => { + vi.restoreAllMocks(); + for (const agentDir of tempAgentDirs.splice(0)) { + fs.rmSync(agentDir, { recursive: true, force: true }); + } +}); + +describe("agent base preset detection", () => { + it("loads the recorded agent policy and distinguishes matching preset names (#9079)", () => { + const agent = createAgentFixture(); + vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "alpha", agent } as never); + + expect(isAgentBasePreset("alpha", "github")).toBe(true); + expect(isAgentBasePreset("alpha", "slack")).toBe(false); + }); + + it("recognizes the Hermes base policy when its preset name also exists in the catalog (#9079)", () => { + const hermesPolicy = fs.readFileSync( + path.join(AGENTS_DIR, "hermes", "policy-additions.yaml"), + "utf8", + ); + const agent = createAgentFixture(hermesPolicy); + vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "hermes", agent } as never); + + expect(isAgentBasePreset("hermes", "pypi")).toBe(true); + }); +}); diff --git a/src/lib/policy/context-builder.ts b/src/lib/policy/context-builder.ts index 22511c3f9ae..7c2ba5e0013 100644 --- a/src/lib/policy/context-builder.ts +++ b/src/lib/policy/context-builder.ts @@ -6,6 +6,7 @@ import { getBaselineExclusionRuntimeStatus, getGatewayPresets, getPresetEndpoints, + isAgentBasePreset, listCustomPresets, listPresets, loadPresetForSandbox, @@ -27,6 +28,7 @@ export type PolicyContextPresetVerification = | "verified" | "registry-only" | "gateway-only" + | "agent-base" | "gateway-unavailable"; export interface PolicyContextPreset { @@ -42,9 +44,9 @@ export interface PolicyContextPreset { source: "builtin" | "custom"; /** * Source-of-truth state for whether this preset is enforced by the - * OpenShell gateway. `verified` and `gateway-only` are based on a live - * gateway probe; `registry-only` and `gateway-unavailable` indicate the - * agent cannot trust this preset as enforced policy. + * OpenShell gateway. `verified`, `gateway-only`, and `agent-base` are + * based on a live gateway probe. `registry-only` and `gateway-unavailable` + * indicate that the agent cannot trust this preset as enforced policy. */ verification: PolicyContextPresetVerification; } @@ -179,15 +181,28 @@ function partitionPresets( const unapplied: PolicyContextPreset[] = []; for (const info of builtin) { const isApplied = applied.has(info.name); - const verification = resolveVerification(info.name, isApplied, gatewayPresets); - const onGatewayOnly = !isApplied && verification === "gateway-only"; + let verification = resolveVerification(info.name, isApplied, gatewayPresets); + // A gateway-only catalog preset whose name collides with an agent + // base-policy addition (e.g. Hermes `pypi`) is enforced by the agent's own + // base policy, not registry drift. Classify it as `agent-base` so it is not + // reported as drift and does not steer operators toward an unnecessary + // `policy add`. + // The apply path already prefers the agent-specific policy content, but it + // would record the preset as operator-applied (#9079). Sibling base additions + // with no catalog entry are never iterated here, so this only corrects the + // incidental name-collision case. + if (!isApplied && verification === "gateway-only" && isAgentBasePreset(sandboxName, info.name)) { + verification = "agent-base"; + } + const enforcedNotApplied = + !isApplied && (verification === "gateway-only" || verification === "agent-base"); const entry = presetEntry( info, "builtin", loadPresetForSandbox(sandboxName, info.name), verification, ); - if (isApplied || onGatewayOnly) { + if (isApplied || enforcedNotApplied) { active.push(entry); } else { unapplied.push(entry); @@ -321,10 +336,11 @@ function probeGatewayPresets( * with a {@link PolicyContextPresetVerification} state: `verified` when * the gateway snapshot agrees, `registry-only` when the gateway does * not enforce the preset (drift), `gateway-only` when the gateway - * enforces something the registry does not list, or - * `gateway-unavailable` when no probe is available. Callers that - * require a trusted "is this host actually allowed?" answer must look - * at `verification === "verified"`; everything else is advisory. + * enforces something the registry does not list, `agent-base` when the + * gateway enforces an agent base-policy preset, or `gateway-unavailable` + * when no probe is available. Callers that require a trusted "is this host + * actually allowed?" answer must accept `verified`, `gateway-only`, and + * `agent-base` as gateway-confirmed states. * * - Host stems are extracted by {@link hostStemsFromContent}, which * redacts RFC1918, loopback, link-local, metadata, and internal-DNS @@ -384,6 +400,8 @@ function verificationTag(verification: PolicyContextPresetVerification): string return "registry-only (gateway does not enforce)"; case "gateway-only": return "gateway-only (not in local registry)"; + case "agent-base": + return "agent-base (enforced by the agent's base policy; not user-applied; `policy add` is unnecessary)"; case "gateway-unavailable": return "gateway-unavailable"; } @@ -507,7 +525,7 @@ export function renderPolicyContextMarkdown(ctx: PolicyContext): string { ); lines.push(""); lines.push( - "Preset status reflects registry vs gateway agreement and is one of `verified`, `registry-only`, `gateway-only`, or `gateway-unavailable`. Treat anything other than `verified` as advisory; an agent must not assume the gateway enforces the listed hosts.", + "Preset status reflects registry and gateway agreement. `verified`, `gateway-only`, and `agent-base` mean the gateway confirms enforcement. `agent-base` identifies a preset from the agent's base policy rather than a user-applied preset. It is active, not drift, and does not need `policy add`. Treat `registry-only` and `gateway-unavailable` as advisory because the gateway has not confirmed the listed hosts.", ); lines.push(""); lines.push(`Generated at ${ctx.generatedAt}.`); diff --git a/src/lib/policy/context.test.ts b/src/lib/policy/context.test.ts index 0518b4f4812..02120c14319 100644 --- a/src/lib/policy/context.test.ts +++ b/src/lib/policy/context.test.ts @@ -13,6 +13,7 @@ vi.mock(".", () => ({ getBaselineExclusionRuntimeStatus: vi.fn(() => "excluded"), getPresetEndpoints: vi.fn(), getGatewayPresets: vi.fn(() => null), + isAgentBasePreset: vi.fn(() => false), listCustomPresets: vi.fn(), listPresets: vi.fn(), loadPreset: vi.fn(), @@ -103,6 +104,8 @@ function resetMocks() { vi.mocked(policies.getPresetEndpoints).mockReset(); vi.mocked(policies.getGatewayPresets).mockReset(); vi.mocked(policies.getGatewayPresets).mockReturnValue(null); + vi.mocked(policies.isAgentBasePreset).mockReset(); + vi.mocked(policies.isAgentBasePreset).mockReturnValue(false); vi.mocked(registry.getBaselineExclusions).mockReset(); vi.mocked(registry.getBaselineExclusions).mockReturnValue([]); vi.mocked(policies.getBaselineExclusionRuntimeStatus).mockReset(); @@ -170,6 +173,43 @@ describe("buildPolicyContext", () => { expect(ctx.knownUnappliedPresets.some((p) => p.name === "github")).toBe(false); }); + it("classifies a gateway-enforced agent-base preset as `agent-base`, not gateway-only drift (#9079)", () => { + resetMocks(); + mockBuiltinPresets(); + stubTier(); + // Neither preset applied by the user; both enforced by the gateway. Only + // `github` is an agent base-policy addition. The other must stay + // gateway-only so genuine drift is still reported. + stubRegistry({ policies: [], policyTier: "restricted" }); + vi.mocked(policies.isAgentBasePreset).mockImplementation( + (_sandboxName: string, name: string) => name === "github", + ); + + const ctx = buildPolicyContext(SANDBOX, { gatewayPresets: ["slack", "github"] }); + + const github = ctx.activePresets.find((p) => p.name === "github"); + const slack = ctx.activePresets.find((p) => p.name === "slack"); + expect(github?.verification).toBe("agent-base"); + expect(slack?.verification).toBe("gateway-only"); + // Agent-base preset is active (enforced), never suggested for `policy add`. + expect(ctx.knownUnappliedPresets.some((p) => p.name === "github")).toBe(false); + }); + + it("does not reclassify an applied preset as agent-base even when the agent defines it (#9079)", () => { + resetMocks(); + mockBuiltinPresets(); + stubTier(); + // `github` is both user-applied and an agent base addition; the user + // intent (applied + enforced) must remain `verified`, not `agent-base`. + stubRegistry({ policies: ["github"], policyTier: "restricted" }); + vi.mocked(policies.isAgentBasePreset).mockReturnValue(true); + + const ctx = buildPolicyContext(SANDBOX, { gatewayPresets: ["github"] }); + + const github = ctx.activePresets.find((p) => p.name === "github"); + expect(github?.verification).toBe("verified"); + }); + it("redacts internal hostnames and IP ranges from allowedHostCategories and counts the drop", () => { resetMocks(); mockBuiltinPresets(); @@ -437,7 +477,57 @@ describe("renderPolicyContextMarkdown", () => { expect(md).not.toMatch(/network_policies:/); }); - it("renders the verification status alongside each active preset", () => { + it.each([ + { + status: "verified", + applied: ["slack"], + gatewayPresets: ["slack"], + agentBase: false, + }, + { + status: "registry-only", + applied: ["slack"], + gatewayPresets: [], + agentBase: false, + }, + { + status: "gateway-only", + applied: [], + gatewayPresets: ["slack"], + agentBase: false, + }, + { + status: "agent-base", + applied: [], + gatewayPresets: ["slack"], + agentBase: true, + }, + { + status: "gateway-unavailable", + applied: ["slack"], + gatewayPresets: null, + agentBase: false, + }, + ])("renders the $status verification status (#9079)", ({ + status, + applied, + gatewayPresets, + agentBase, + }) => { + resetMocks(); + mockBuiltinPresets(); + stubTier(); + stubRegistry({ policies: applied, policyTier: "balanced" }); + vi.mocked(policies.isAgentBasePreset).mockReturnValue(agentBase); + + const md = renderPolicyContextMarkdown( + buildPolicyContext(SANDBOX, { gatewayPresets }), + ); + + expect(md).toContain(`status: ${status}`); + }); + + it("states which verification statuses confirm gateway enforcement (#9079)", () => { resetMocks(); mockBuiltinPresets(); stubTier(); @@ -446,7 +536,13 @@ describe("renderPolicyContextMarkdown", () => { const md = renderPolicyContextMarkdown( buildPolicyContext(SANDBOX, { gatewayPresets: ["slack"] }), ); - expect(md).toContain("status: verified"); + + expect(md).toContain( + "`verified`, `gateway-only`, and `agent-base` mean the gateway confirms enforcement", + ); + expect(md).toContain( + "Treat `registry-only` and `gateway-unavailable` as advisory because the gateway has not confirmed the listed hosts", + ); }); it("discloses excluded baseline entries and their support impact (#7194)", () => { diff --git a/src/lib/policy/failure-classifier.test.ts b/src/lib/policy/failure-classifier.test.ts index afce73624e6..56f818e26ce 100644 --- a/src/lib/policy/failure-classifier.test.ts +++ b/src/lib/policy/failure-classifier.test.ts @@ -13,6 +13,7 @@ vi.mock(".", () => ({ getPresetEndpoints: vi.fn(), getGatewayPresets: vi.fn(() => null), getSandboxBaselineEntryDigest: vi.fn(() => null), + isAgentBasePreset: vi.fn(() => false), listCustomPresets: vi.fn(), listPresets: vi.fn(), loadPreset: vi.fn(), @@ -103,6 +104,8 @@ function resetMocks() { vi.mocked(policies.getPresetEndpoints).mockReset(); vi.mocked(policies.getGatewayPresets).mockReset(); vi.mocked(policies.getGatewayPresets).mockReturnValue(null); + vi.mocked(policies.isAgentBasePreset).mockReset(); + vi.mocked(policies.isAgentBasePreset).mockReturnValue(false); vi.mocked(getTier).mockReset(); } @@ -299,6 +302,28 @@ describe("classifyAccessFailure", () => { expect(result.reason).toContain("upstream"); }); + it("treats an agent-base preset host hitting a network-block code as high-confidence upstream-unknown (#9079)", () => { + resetMocks(); + mockBuiltinPresets(); + stubTier(); + // `slack` is enforced by the gateway via the agent base policy, not + // user-applied. Verification resolves to `agent-base`, which is enforced, + // so a block code is upstream (like `verified`), not a policy denial. + stubRegistry({ policies: [], policyTier: "restricted" }); + vi.mocked(policies.isAgentBasePreset).mockReturnValue(true); + + const result = classifyAccessFailure({ + sandboxName: SANDBOX, + host: "api.slack.com", + error: { code: "EHOSTUNREACH" }, + gatewayPresets: ["slack"], + }); + + expect(result.kind).toBe("unknown"); + expect(result.matchedPreset).toBe("slack"); + expect(result.confidence).toBe("high"); + }); + it.each([ "EHOSTUNREACH", "ENETUNREACH", diff --git a/src/lib/policy/failure-classifier.ts b/src/lib/policy/failure-classifier.ts index 4ea8721b3f6..e8e26efa8ef 100644 --- a/src/lib/policy/failure-classifier.ts +++ b/src/lib/policy/failure-classifier.ts @@ -93,7 +93,11 @@ function findMatchingPreset( } function isVerified(preset: PolicyContextPreset): boolean { - return preset.verification === "verified" || preset.verification === "gateway-only"; + return ( + preset.verification === "verified" || + preset.verification === "gateway-only" || + preset.verification === "agent-base" + ); } function verificationNote(preset: PolicyContextPreset): string { diff --git a/src/lib/policy/index.ts b/src/lib/policy/index.ts index ddb1fdb6126..7f33f2c1d3c 100644 --- a/src/lib/policy/index.ts +++ b/src/lib/policy/index.ts @@ -297,6 +297,22 @@ function loadAgentPresetContent( } } +/** + * True when `presetName` is supplied by the sandbox agent's base policy + * (`agents//policy-additions.yaml`) rather than only by the built-in + * catalog. Used to distinguish an agent base-policy entry that the gateway + * enforces (for example, Hermes `pypi`) from genuine registry drift. + * `policy explain` can then avoid an unnecessary `policy add`, which would + * record the preset as operator-applied even though the apply path already + * prefers the agent-specific policy content (#9079). Best-effort: any load + * failure resolves to `false`, preserving the pre-existing gateway-only + * classification. + */ +function isAgentBasePreset(sandboxName: string, presetName: string): boolean { + const builtinPresetContent = loadCentralPreset(presetName); + return loadAgentPresetContent(sandboxName, presetName, builtinPresetContent ?? "") !== null; +} + function loadPresetForSandbox(sandboxName: string, presetName: string): string | null { let sandboxAgent: string | null = null; try { @@ -2682,6 +2698,7 @@ export { getPresetValidationWarning, getSandboxBaselineEntry, getSandboxBaselineEntryDigest, + isAgentBasePreset, isMessagingChannelPolicyPreset, listCustomPresets, listPresets,