Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/network-policy/explain-network-policy-to-agents.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ Each active preset includes a `verification` value:
| `verified` | The registry lists the preset, and the gateway confirms enforcement. |
| `registry-only` | The registry lists the preset, but the gateway does not enforce it. Treat the allowed hosts as unverified. |
| `gateway-only` | The gateway enforces a preset that the registry does not list. |
| `agent-base` | The gateway enforces this preset because it belongs to the agent's own base policy (`agents/<agent>/policy-additions.yaml`), not because the operator applied it. It is active, not drift. `policy add` is unnecessary and would record the preset as operator-applied. |
| `gateway-unavailable` | NemoClaw could not probe the gateway. Treat the report as advisory until the gateway is reachable. |

## Classify a Failed Request
Expand All @@ -81,7 +82,7 @@ The classifier evaluates conditions in this order:

Network-block error codes include `EHOSTUNREACH`, `ENETUNREACH`, `ENOTFOUND`, `ECONNREFUSED`, `ETIMEDOUT`, and `EAI_AGAIN`.
A block code on a host from a `registry-only` or `gateway-unavailable` preset produces a low-confidence policy verdict.
A block code on a host from a verified preset stays `unknown` because the gateway already confirmed enforcement.
A block code on a host from a `verified`, `gateway-only`, or `agent-base` preset stays `unknown` with high confidence because the gateway confirmed enforcement.

Each verdict includes `confidence` set to `high` or `low`.
Low confidence means the agent must report multiple possibilities instead of treating one next step as authoritative.
Expand Down
69 changes: 69 additions & 0 deletions src/lib/policy/agent-base-preset.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { randomUUID } from "node:crypto";
import fs from "node:fs";
import path from "node:path";

import { afterEach, describe, expect, it, vi } from "vitest";

import { AGENTS_DIR } from "../agent/defs";
import * as registry from "../state/registry";
import { isAgentBasePreset } from "./index";

const tempAgentDirs: string[] = [];

function createAgentFixture(policyAdditions?: string): string {
const agentName = `agent-base-preset-${randomUUID()}`;
const agentDir = path.join(AGENTS_DIR, agentName);
tempAgentDirs.push(agentDir);
fs.mkdirSync(agentDir, { recursive: true });
fs.writeFileSync(
path.join(agentDir, "manifest.yaml"),
`name: ${agentName}\ndisplay_name: Agent Base Preset Fixture\n`,
);
fs.writeFileSync(
path.join(agentDir, "policy-additions.yaml"),
policyAdditions ??
`version: 1
network_policies:
github:
name: github
endpoints:
- host: api.github.com
port: 443
access: full
binaries:
- path: /usr/bin/git
`,
);
return agentName;
}

afterEach(() => {
vi.restoreAllMocks();
for (const agentDir of tempAgentDirs.splice(0)) {
fs.rmSync(agentDir, { recursive: true, force: true });
}
});

describe("agent base preset detection", () => {
it("loads the recorded agent policy and distinguishes matching preset names (#9079)", () => {
const agent = createAgentFixture();
vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "alpha", agent } as never);

expect(isAgentBasePreset("alpha", "github")).toBe(true);
expect(isAgentBasePreset("alpha", "slack")).toBe(false);
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.

it("recognizes the Hermes base policy when its preset name also exists in the catalog (#9079)", () => {
const hermesPolicy = fs.readFileSync(
path.join(AGENTS_DIR, "hermes", "policy-additions.yaml"),
"utf8",
);
const agent = createAgentFixture(hermesPolicy);
vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "hermes", agent } as never);

expect(isAgentBasePreset("hermes", "pypi")).toBe(true);
});
});
40 changes: 29 additions & 11 deletions src/lib/policy/context-builder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
getBaselineExclusionRuntimeStatus,
getGatewayPresets,
getPresetEndpoints,
isAgentBasePreset,
listCustomPresets,
listPresets,
loadPresetForSandbox,
Expand All @@ -27,6 +28,7 @@ export type PolicyContextPresetVerification =
| "verified"
| "registry-only"
| "gateway-only"
| "agent-base"
| "gateway-unavailable";

export interface PolicyContextPreset {
Expand All @@ -42,9 +44,9 @@ export interface PolicyContextPreset {
source: "builtin" | "custom";
/**
* Source-of-truth state for whether this preset is enforced by the
* OpenShell gateway. `verified` and `gateway-only` are based on a live
* gateway probe; `registry-only` and `gateway-unavailable` indicate the
* agent cannot trust this preset as enforced policy.
* OpenShell gateway. `verified`, `gateway-only`, and `agent-base` are
* based on a live gateway probe. `registry-only` and `gateway-unavailable`
* indicate that the agent cannot trust this preset as enforced policy.
*/
verification: PolicyContextPresetVerification;
}
Expand Down Expand Up @@ -179,15 +181,28 @@ function partitionPresets(
const unapplied: PolicyContextPreset[] = [];
for (const info of builtin) {
const isApplied = applied.has(info.name);
const verification = resolveVerification(info.name, isApplied, gatewayPresets);
const onGatewayOnly = !isApplied && verification === "gateway-only";
let verification = resolveVerification(info.name, isApplied, gatewayPresets);
// A gateway-only catalog preset whose name collides with an agent
// base-policy addition (e.g. Hermes `pypi`) is enforced by the agent's own
// base policy, not registry drift. Classify it as `agent-base` so it is not
// reported as drift and does not steer operators toward an unnecessary
// `policy add`.
// The apply path already prefers the agent-specific policy content, but it
// would record the preset as operator-applied (#9079). Sibling base additions
// with no catalog entry are never iterated here, so this only corrects the
// incidental name-collision case.
if (!isApplied && verification === "gateway-only" && isAgentBasePreset(sandboxName, info.name)) {
verification = "agent-base";
}
const enforcedNotApplied =
!isApplied && (verification === "gateway-only" || verification === "agent-base");
const entry = presetEntry(
info,
"builtin",
loadPresetForSandbox(sandboxName, info.name),
verification,
);
if (isApplied || onGatewayOnly) {
if (isApplied || enforcedNotApplied) {
active.push(entry);
} else {
unapplied.push(entry);
Expand Down Expand Up @@ -321,10 +336,11 @@ function probeGatewayPresets(
* with a {@link PolicyContextPresetVerification} state: `verified` when
* the gateway snapshot agrees, `registry-only` when the gateway does
* not enforce the preset (drift), `gateway-only` when the gateway
* enforces something the registry does not list, or
* `gateway-unavailable` when no probe is available. Callers that
* require a trusted "is this host actually allowed?" answer must look
* at `verification === "verified"`; everything else is advisory.
* enforces something the registry does not list, `agent-base` when the
* gateway enforces an agent base-policy preset, or `gateway-unavailable`
* when no probe is available. Callers that require a trusted "is this host
* actually allowed?" answer must accept `verified`, `gateway-only`, and
* `agent-base` as gateway-confirmed states.
*
* - Host stems are extracted by {@link hostStemsFromContent}, which
* redacts RFC1918, loopback, link-local, metadata, and internal-DNS
Expand Down Expand Up @@ -384,6 +400,8 @@ function verificationTag(verification: PolicyContextPresetVerification): string
return "registry-only (gateway does not enforce)";
case "gateway-only":
return "gateway-only (not in local registry)";
case "agent-base":
return "agent-base (enforced by the agent's base policy; not user-applied; `policy add` is unnecessary)";
case "gateway-unavailable":
return "gateway-unavailable";
}
Expand Down Expand Up @@ -507,7 +525,7 @@ export function renderPolicyContextMarkdown(ctx: PolicyContext): string {
);
lines.push("");
lines.push(
"Preset status reflects registry vs gateway agreement and is one of `verified`, `registry-only`, `gateway-only`, or `gateway-unavailable`. Treat anything other than `verified` as advisory; an agent must not assume the gateway enforces the listed hosts.",
"Preset status reflects registry and gateway agreement. `verified`, `gateway-only`, and `agent-base` mean the gateway confirms enforcement. `agent-base` identifies a preset from the agent's base policy rather than a user-applied preset. It is active, not drift, and does not need `policy add`. Treat `registry-only` and `gateway-unavailable` as advisory because the gateway has not confirmed the listed hosts.",
);
lines.push("");
lines.push(`Generated at ${ctx.generatedAt}.`);
Expand Down
100 changes: 98 additions & 2 deletions src/lib/policy/context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ vi.mock(".", () => ({
getBaselineExclusionRuntimeStatus: vi.fn(() => "excluded"),
getPresetEndpoints: vi.fn(),
getGatewayPresets: vi.fn(() => null),
isAgentBasePreset: vi.fn(() => false),
listCustomPresets: vi.fn(),
listPresets: vi.fn(),
loadPreset: vi.fn(),
Expand Down Expand Up @@ -103,6 +104,8 @@ function resetMocks() {
vi.mocked(policies.getPresetEndpoints).mockReset();
vi.mocked(policies.getGatewayPresets).mockReset();
vi.mocked(policies.getGatewayPresets).mockReturnValue(null);
vi.mocked(policies.isAgentBasePreset).mockReset();
vi.mocked(policies.isAgentBasePreset).mockReturnValue(false);
vi.mocked(registry.getBaselineExclusions).mockReset();
vi.mocked(registry.getBaselineExclusions).mockReturnValue([]);
vi.mocked(policies.getBaselineExclusionRuntimeStatus).mockReset();
Expand Down Expand Up @@ -170,6 +173,43 @@ describe("buildPolicyContext", () => {
expect(ctx.knownUnappliedPresets.some((p) => p.name === "github")).toBe(false);
});

it("classifies a gateway-enforced agent-base preset as `agent-base`, not gateway-only drift (#9079)", () => {
resetMocks();
mockBuiltinPresets();
stubTier();
// Neither preset applied by the user; both enforced by the gateway. Only
// `github` is an agent base-policy addition. The other must stay
// gateway-only so genuine drift is still reported.
stubRegistry({ policies: [], policyTier: "restricted" });
vi.mocked(policies.isAgentBasePreset).mockImplementation(
(_sandboxName: string, name: string) => name === "github",
);

const ctx = buildPolicyContext(SANDBOX, { gatewayPresets: ["slack", "github"] });

const github = ctx.activePresets.find((p) => p.name === "github");
const slack = ctx.activePresets.find((p) => p.name === "slack");
expect(github?.verification).toBe("agent-base");
expect(slack?.verification).toBe("gateway-only");
// Agent-base preset is active (enforced), never suggested for `policy add`.
expect(ctx.knownUnappliedPresets.some((p) => p.name === "github")).toBe(false);
});

it("does not reclassify an applied preset as agent-base even when the agent defines it (#9079)", () => {
resetMocks();
mockBuiltinPresets();
stubTier();
// `github` is both user-applied and an agent base addition; the user
// intent (applied + enforced) must remain `verified`, not `agent-base`.
stubRegistry({ policies: ["github"], policyTier: "restricted" });
vi.mocked(policies.isAgentBasePreset).mockReturnValue(true);

const ctx = buildPolicyContext(SANDBOX, { gatewayPresets: ["github"] });

const github = ctx.activePresets.find((p) => p.name === "github");
expect(github?.verification).toBe("verified");
});

it("redacts internal hostnames and IP ranges from allowedHostCategories and counts the drop", () => {
resetMocks();
mockBuiltinPresets();
Expand Down Expand Up @@ -437,7 +477,57 @@ describe("renderPolicyContextMarkdown", () => {
expect(md).not.toMatch(/network_policies:/);
});

it("renders the verification status alongside each active preset", () => {
it.each([
{
status: "verified",
applied: ["slack"],
gatewayPresets: ["slack"],
agentBase: false,
},
{
status: "registry-only",
applied: ["slack"],
gatewayPresets: [],
agentBase: false,
},
{
status: "gateway-only",
applied: [],
gatewayPresets: ["slack"],
agentBase: false,
},
{
status: "agent-base",
applied: [],
gatewayPresets: ["slack"],
agentBase: true,
},
{
status: "gateway-unavailable",
applied: ["slack"],
gatewayPresets: null,
agentBase: false,
},
])("renders the $status verification status (#9079)", ({
status,
applied,
gatewayPresets,
agentBase,
}) => {
resetMocks();
mockBuiltinPresets();
stubTier();
stubRegistry({ policies: applied, policyTier: "balanced" });
vi.mocked(policies.isAgentBasePreset).mockReturnValue(agentBase);

const md = renderPolicyContextMarkdown(
buildPolicyContext(SANDBOX, { gatewayPresets }),
);

expect(md).toContain(`status: ${status}`);
});

it("states which verification statuses confirm gateway enforcement (#9079)", () => {
resetMocks();
mockBuiltinPresets();
stubTier();
Expand All @@ -446,7 +536,13 @@ describe("renderPolicyContextMarkdown", () => {
const md = renderPolicyContextMarkdown(
buildPolicyContext(SANDBOX, { gatewayPresets: ["slack"] }),
);
expect(md).toContain("status: verified");

expect(md).toContain(
"`verified`, `gateway-only`, and `agent-base` mean the gateway confirms enforcement",
);
expect(md).toContain(
"Treat `registry-only` and `gateway-unavailable` as advisory because the gateway has not confirmed the listed hosts",
);
});

it("discloses excluded baseline entries and their support impact (#7194)", () => {
Expand Down
25 changes: 25 additions & 0 deletions src/lib/policy/failure-classifier.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ vi.mock(".", () => ({
getPresetEndpoints: vi.fn(),
getGatewayPresets: vi.fn(() => null),
getSandboxBaselineEntryDigest: vi.fn(() => null),
isAgentBasePreset: vi.fn(() => false),
listCustomPresets: vi.fn(),
listPresets: vi.fn(),
loadPreset: vi.fn(),
Expand Down Expand Up @@ -103,6 +104,8 @@ function resetMocks() {
vi.mocked(policies.getPresetEndpoints).mockReset();
vi.mocked(policies.getGatewayPresets).mockReset();
vi.mocked(policies.getGatewayPresets).mockReturnValue(null);
vi.mocked(policies.isAgentBasePreset).mockReset();
vi.mocked(policies.isAgentBasePreset).mockReturnValue(false);
vi.mocked(getTier).mockReset();
}

Expand Down Expand Up @@ -299,6 +302,28 @@ describe("classifyAccessFailure", () => {
expect(result.reason).toContain("upstream");
});

it("treats an agent-base preset host hitting a network-block code as high-confidence upstream-unknown (#9079)", () => {
resetMocks();
mockBuiltinPresets();
stubTier();
// `slack` is enforced by the gateway via the agent base policy, not
// user-applied. Verification resolves to `agent-base`, which is enforced,
// so a block code is upstream (like `verified`), not a policy denial.
stubRegistry({ policies: [], policyTier: "restricted" });
vi.mocked(policies.isAgentBasePreset).mockReturnValue(true);

const result = classifyAccessFailure({
sandboxName: SANDBOX,
host: "api.slack.com",
error: { code: "EHOSTUNREACH" },
gatewayPresets: ["slack"],
});

expect(result.kind).toBe("unknown");
expect(result.matchedPreset).toBe("slack");
expect(result.confidence).toBe("high");
});

it.each([
"EHOSTUNREACH",
"ENETUNREACH",
Expand Down
6 changes: 5 additions & 1 deletion src/lib/policy/failure-classifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,11 @@ function findMatchingPreset(
}

function isVerified(preset: PolicyContextPreset): boolean {
return preset.verification === "verified" || preset.verification === "gateway-only";
return (
preset.verification === "verified" ||
preset.verification === "gateway-only" ||
preset.verification === "agent-base"
);
}

function verificationNote(preset: PolicyContextPreset): string {
Expand Down
Loading
Loading