From df95ac2a3154a98898c08be065e0e3aba14d1fb1 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Sat, 15 Aug 2026 14:05:47 +0200 Subject: [PATCH 1/2] feat(operator): let it test-drive an agent or group by actually talking to it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operator could build an agent and never exercise one. Asked to check its own creation it answered "I don't have a start conversation tool" - accurate, and useless to the admin who had just approved the build. "Deployed" only means the config loaded; it says nothing about whether the LLM call, the tool wiring or the vault key resolve at runtime. Grants the runtime conversation endpoints the Manager's own chat uses: start, say, read-back, and the group equivalents. No new tool is written - the operator's tools are generated from the OpenAPI spec, so a capability IS an allow-list entry. The POSTs go in WRITE_ENDPOINTS, not READ_ENDPOINTS. They change no configuration, but grantsWriteCapability keys off the method: with them among the reads, a read_only operator lost its "you are read-only" rule and gained the six write rules - describing a boundary it was not behind. The existing tests caught it. READ_ENDPOINTS means "GETs", not "harmless". Consequence, accepted: a read-only operator cannot test-drive, which is coherent since it cannot create the conversation record either. Every send pauses for approval like any other write - sending a message as the admin is a decision they should see - and the agent under test keeps its OWN gate. exempt stays ["http.get:*"]. Excluded and pinned by test: /resume (self-approval - a complete escape from the gate), /state, /cancel, /endConversation, /undo, /redo. See planning/operator-write-scope-plan.md §5. Prompt guidance is gated on the endpoints actually being granted, and states the rule that matters: a conversation coming back AWAITING_HUMAN means the agent under test paused on its OWN gate - a PASS, not a failure. --- .../operator/__tests__/system-prompt.test.ts | 92 ++++++++++++++++++- .../operator/__tests__/tool-scopes.test.ts | 9 ++ src/lib/operator/system-prompt.ts | 37 +++++++- src/lib/operator/tool-scopes.ts | 49 ++++++++++ 4 files changed, 185 insertions(+), 2 deletions(-) diff --git a/src/lib/operator/__tests__/system-prompt.test.ts b/src/lib/operator/__tests__/system-prompt.test.ts index 677e6bca..a02263f3 100644 --- a/src/lib/operator/__tests__/system-prompt.test.ts +++ b/src/lib/operator/__tests__/system-prompt.test.ts @@ -6,7 +6,13 @@ import { defaultOperatorPromptBody, safetyPreambleForScope, } from "../system-prompt"; -import { READ_ENDPOINTS, WRITE_ENDPOINTS, endpointsForScope } from "../tool-scopes"; +import { + READ_ENDPOINTS, + WRITE_ENDPOINTS, + endpointsForScope, + buildToolApprovals, + grantsConversationTesting, +} from "../tool-scopes"; /** * A granted set that contains a write. @@ -397,3 +403,87 @@ describe("prompt corrections from dev testing", () => { expect(defaultOperatorPromptBody("read_only")).not.toContain("setupAgent essentials"); }); }); + +/** + * Test-drive: the operator could build an agent but never exercise one. Asked to + * check its own creation it answered "I don't have a start conversation tool" — + * accurate, and useless to the admin who had just approved the build. + */ +describe("test-drive: talking to another agent", () => { + it("grants start + say to read_write, and the read-back to both", () => { + const write = new Set(endpointsForScope("read_write")); + expect(write.has("POST /agents/{agentId}/start")).toBe(true); + expect(write.has("POST /agents/{conversationId}")).toBe(true); + expect(write.has("POST /groups/{groupId}/conversations")).toBe(true); + + // The POSTs are writes by method, so they sit in WRITE_ENDPOINTS: putting + // them in READ_ENDPOINTS flipped read_only into the write branch of the + // safety preamble. Reading a conversation back is a plain GET and is + // granted to both scopes. + for (const scope of ["read_only", "read_write"] as const) { + expect(new Set(endpointsForScope(scope)).has("GET /agents/{conversationId}")).toBe(true); + } + const read = new Set(endpointsForScope("read_only")); + expect(read.has("POST /agents/{agentId}/start")).toBe(false); + expect(read.has("POST /agents/{conversationId}")).toBe(false); + }); + + it("leaves read_only genuinely read-only — the regression the tests caught", () => { + expect(safetyPreambleForScope("read_only")).toContain("You are read-only"); + expect(defaultOperatorPromptBody("read_only")).not.toContain("Testing an agent"); + }); + + /** + * THE regression guard. `/resume` would let the operator approve its own + * pauses — a complete escape from the gate — and the other three let it + * rewrite or discard a conversation's lifecycle. All are excluded by + * planning/operator-write-scope-plan.md §5. + */ + it("never grants resume, state, cancel or end — for any scope", () => { + for (const scope of ["read_only", "read_write"] as const) { + const set = new Set(endpointsForScope(scope)); + for (const forbidden of [ + "POST /agents/{conversationId}/resume", + "PATCH /agents/{conversationId}/state", + "POST /agents/{conversationId}/cancel", + "POST /agents/{conversationId}/endConversation", + "POST /agents/{conversationId}/undo", + "POST /agents/{conversationId}/redo", + ]) { + expect(set.has(forbidden)).toBe(false); + } + } + }); + + it("keeps the gate intact — the new POSTs are approved, never exempt", () => { + // Adding a conversation POST to `exempt` would be the first hole ever + // punched in http.post:*, and verifyGateInstalled would reject it anyway. + expect(buildToolApprovals().exempt).toEqual(["http.get:*"]); + expect(buildToolApprovals().requireApproval).toContain("http.post:*"); + }); + + it("tells the operator that an AWAITING_HUMAN reply is a PASS, not a failure", () => { + // An agent with its own gate is supposed to stop. Reading that as broken + // would report a correctly-configured agent as failing. + const body = defaultOperatorPromptBody("read_write"); + expect(body).toContain("Testing an agent"); + expect(body).toMatch(/paused on ITS OWN approval gate/); + expect(body).toMatch(/That is a PASS/); + expect(body).toMatch(/cannot approve on another\s+agent's behalf/); + }); + + it("says nothing about test-driving when the endpoints are not granted", () => { + // The module's rule: the prompt may never describe a capability the agent + // lacks. Pass a set with the reads but neither conversation POST. + const body = buildOperatorPromptBody(["GET /agentstore/agents/descriptors"]); + expect(body).not.toContain("Testing an agent"); + }); + + it("requires BOTH start and say — start alone proves nothing", () => { + expect(grantsConversationTesting(["POST /agents/{agentId}/start"])).toBe(false); + expect(grantsConversationTesting(["POST /agents/{conversationId}"])).toBe(false); + expect( + grantsConversationTesting(["POST /agents/{agentId}/start", "POST /agents/{conversationId}"]), + ).toBe(true); + }); +}); diff --git a/src/lib/operator/__tests__/tool-scopes.test.ts b/src/lib/operator/__tests__/tool-scopes.test.ts index e4cd3811..1d6e2071 100644 --- a/src/lib/operator/__tests__/tool-scopes.test.ts +++ b/src/lib/operator/__tests__/tool-scopes.test.ts @@ -94,6 +94,15 @@ describe("tool-scopes", () => { // addition here is exactly as dangerous as a silent removal from an // allow-list — this test catches either direction. expect(WRITE_ENDPOINTS).toEqual([ + // Test-drive. Writes by METHOD only — they create a conversation and a + // message, never a configuration change. Here rather than in + // READ_ENDPOINTS because grantsWriteCapability keys off the method, and + // a POST among the reads flipped read_only into the write branch of the + // safety preamble. `/resume`, `/state`, `/cancel` and `/endConversation` + // are excluded on purpose — see TEST_DRIVE_WRITES. + "POST /agents/{agentId}/start", + "POST /agents/{conversationId}", + "POST /groups/{groupId}/conversations", "PATCH /descriptorstore/descriptors/{id}", "POST /administration/{environment}/deploy/{agentId}", "POST /administration/{environment}/undeploy/{agentId}", diff --git a/src/lib/operator/system-prompt.ts b/src/lib/operator/system-prompt.ts index cd7bbca3..a4332ae3 100644 --- a/src/lib/operator/system-prompt.ts +++ b/src/lib/operator/system-prompt.ts @@ -3,6 +3,7 @@ import { grantsWriteCapability, grantsAgentCreation, grantsAgentModification, + grantsConversationTesting, type OperatorScope, } from "./tool-scopes"; import { MODEL_SUGGESTIONS } from "@/lib/model-suggestions"; @@ -125,7 +126,38 @@ You can: - Read EDDI's own documentation. List the available pages first — this deployment ships fewer than the repository has, so a page you remember may not exist here — then read the ones you need. Prefer citing the docs over - answering "how does EDDI do X?" from memory.`; + answering "how does EDDI do X?" from memory. +- TEST-DRIVE an agent or a group: start a conversation with it, send a message, + and read the reply back. "Deployed" only means it loaded; a test message is + the only thing that proves the LLM call, the tool wiring and any vault key + actually resolve at runtime. After creating or changing an agent, offer this.`; + +/** + * How to use the runtime conversation endpoints well. + * + * Included whenever those endpoints are granted, in BOTH scopes — a read-only + * operator can still be asked "is this agent actually answering?", and that is a + * diagnosis, not a change. + * + * The `AWAITING_HUMAN` rule is the one that matters: an agent with its own + * approval gate is SUPPOSED to stop, and an operator that reads that as a + * failure would report a correctly-configured agent as broken. + */ +const BODY_TEST_DRIVE = `Testing an agent (or a group) by talking to it: +- Start a conversation, send ONE representative message, then read the + conversation back and quote what the agent actually replied. +- Each message you send needs the admin's approval, so make it count: say what + you are about to send and why, and prefer one good test message to a + conversation. +- Use the agent's own environment. An agent deployed to \`test\` is not + reachable in \`production\`, and "no response" from the wrong environment is a + false alarm — check deployment status first if you are unsure. +- If the conversation comes back \`AWAITING_HUMAN\`, the agent you are testing + paused on ITS OWN approval gate. That is a PASS, not a failure: it proves the + gate works. Report which call it stopped on. You cannot approve on another + agent's behalf — say so and let the admin decide. +- If the reply is empty or an error, report it verbatim with the conversation id + rather than re-sending. A failing agent is a finding, not something to retry.`; /** * Architecture background, present in BOTH scopes — a read-only operator @@ -396,6 +428,9 @@ export function buildOperatorPromptBody(endpoints: readonly string[]): string { BODY_ROLE, BODY_ARCHITECTURE, BODY_CHEATSHEET, + // Gated on the endpoints actually granted, not on scope: this module's rule + // is that the prompt can never describe a capability the agent lacks. + ...(grantsConversationTesting(endpoints) ? [BODY_TEST_DRIVE] : []), BODY_APP_CONTEXT, buildModelCatalogueSection(), BODY_STYLE, diff --git a/src/lib/operator/tool-scopes.ts b/src/lib/operator/tool-scopes.ts index 661ece0a..807dc512 100644 --- a/src/lib/operator/tool-scopes.ts +++ b/src/lib/operator/tool-scopes.ts @@ -116,6 +116,11 @@ export const READ_ENDPOINTS: readonly string[] = [ // Conversations "GET /conversationstore/conversations", "GET /conversationstore/conversations/{conversationId}", + // Test-drive, read half: inspect a conversation with another agent. The POSTs + // that CREATE one live in WRITE_ENDPOINTS — see TEST_DRIVE_WRITES. + "GET /agents/{conversationId}", + "GET /agents/{conversationId}/status", + "GET /groups/{groupId}/conversations/{groupConversationId}", // Operations "GET /administration/{environment}/deploymentstatus/{agentId}", "GET /administration/coordinator/status", @@ -251,7 +256,38 @@ export const READ_ENDPOINTS: readonly string[] = [ * unconditionally, anything added here is gated the moment it is added — the * failure mode of forgetting to update a pattern list does not exist. */ +/** + * Starting a conversation with another agent, and saying something into it. + * + * These are POSTs, so they belong in the write set — not because they change any + * configuration (they do not), but because `grantsWriteCapability` and the whole + * gate model key off the method. Putting them in `READ_ENDPOINTS` flipped a + * read-only operator into the write branch of the safety preamble: it lost the + * "you are read-only" rule and gained the six write rules, describing a + * capability boundary it was not actually behind. The tests caught it; the + * lesson is that `READ_ENDPOINTS` means "GETs", not "harmless". + * + * The consequence, accepted deliberately: a read-only operator cannot test-drive + * an agent. That is coherent — it cannot create the conversation record either. + * + * Every call here pauses for human approval like any other write. That is the + * point: sending a message as the admin is a decision they should see, and the + * agent being tested keeps its OWN gate for anything it then does. + * + * Deliberately ABSENT, and they must stay absent: `POST /agents/{id}/resume` + * (the operator approving its own pauses would be a complete escape from the + * gate), `PATCH /agents/{conversationId}/state`, `/cancel`, `/endConversation`, + * `/undo`, `/redo`, and `DELETE …/conversations/{id}` — see + * `planning/operator-write-scope-plan.md` §5 in the EDDI repo. + */ +const TEST_DRIVE_WRITES = [ + "POST /agents/{agentId}/start", + "POST /agents/{conversationId}", + "POST /groups/{groupId}/conversations", +] as const; + export const WRITE_ENDPOINTS: readonly string[] = [ + ...TEST_DRIVE_WRITES, "PATCH /descriptorstore/descriptors/{id}", "POST /administration/{environment}/deploy/{agentId}", "POST /administration/{environment}/undeploy/{agentId}", @@ -394,6 +430,19 @@ export function grantsAgentCreation(endpoints: readonly string[]): boolean { return set.has("POST /administration/agents/setup") || set.has("POST /administration/agents/setup-api"); } +/** + * Whether the granted endpoints let the operator hold a conversation with + * another agent — start one AND say something into it. + * + * Both are required: `start` alone creates an empty conversation and proves + * nothing, which is exactly the useless half-capability the prompt must not + * advertise. + */ +export function grantsConversationTesting(endpoints: readonly string[]): boolean { + const set = new Set(endpoints); + return set.has("POST /agents/{agentId}/start") && set.has("POST /agents/{conversationId}"); +} + /** * Whether the granted endpoints can change an existing agent's behavior, * outputs, tool wiring, or pipeline — any workflow or writable From 4d1343dbfa50af0339ec88bf21d66dfc53bb4aab Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Sat, 15 Aug 2026 14:16:27 +0200 Subject: [PATCH 2/2] fix(operator): the test-drive promise sat in the UNCONDITIONAL prompt section CodeRabbit, Major, and correct: the bullet went into BODY_ROLE, which is always included, so a read_only operator - whose endpoint set excludes both conversation POSTs - was told to start conversations and send messages. That is exactly the contract this module exists to enforce: the prompt may never describe a capability the agent lacks. Moved into BODY_TEST_DRIVE, which is already gated on grantsConversationTesting. The existing assertion would not have caught it - it checked only for the section heading, which was never in BODY_ROLE. It now asserts on every phrase that promises the capability. Verified by mutation: making the section unconditional fails two tests. --- .../operator/__tests__/system-prompt.test.ts | 21 ++++++++++++++++++- src/lib/operator/system-prompt.ts | 11 +++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/lib/operator/__tests__/system-prompt.test.ts b/src/lib/operator/__tests__/system-prompt.test.ts index a02263f3..29dc446a 100644 --- a/src/lib/operator/__tests__/system-prompt.test.ts +++ b/src/lib/operator/__tests__/system-prompt.test.ts @@ -430,7 +430,26 @@ describe("test-drive: talking to another agent", () => { it("leaves read_only genuinely read-only — the regression the tests caught", () => { expect(safetyPreambleForScope("read_only")).toContain("You are read-only"); - expect(defaultOperatorPromptBody("read_only")).not.toContain("Testing an agent"); + }); + + /** + * The prompt must never describe a capability the agent lacks — the whole + * reason this module derives from the endpoint set. A first attempt put the + * test-drive bullet in BODY_ROLE, which is unconditional, so a read_only + * operator was told to start conversations it has no endpoint for. Asserting + * on the section heading alone would NOT have caught that, so this checks + * every phrase that promises the capability. + */ + it("says nothing whatsoever about test-driving in a read_only body", () => { + const body = defaultOperatorPromptBody("read_only"); + for (const promise of [ + "Testing an agent", + "TEST-DRIVE", + "Start a conversation", + "start a conversation with it", + ]) { + expect(body).not.toContain(promise); + } }); /** diff --git a/src/lib/operator/system-prompt.ts b/src/lib/operator/system-prompt.ts index a4332ae3..2cef772b 100644 --- a/src/lib/operator/system-prompt.ts +++ b/src/lib/operator/system-prompt.ts @@ -126,11 +126,7 @@ You can: - Read EDDI's own documentation. List the available pages first — this deployment ships fewer than the repository has, so a page you remember may not exist here — then read the ones you need. Prefer citing the docs over - answering "how does EDDI do X?" from memory. -- TEST-DRIVE an agent or a group: start a conversation with it, send a message, - and read the reply back. "Deployed" only means it loaded; a test message is - the only thing that proves the LLM call, the tool wiring and any vault key - actually resolve at runtime. After creating or changing an agent, offer this.`; + answering "how does EDDI do X?" from memory.`; /** * How to use the runtime conversation endpoints well. @@ -144,6 +140,11 @@ You can: * failure would report a correctly-configured agent as broken. */ const BODY_TEST_DRIVE = `Testing an agent (or a group) by talking to it: +- You can TEST-DRIVE any deployed agent or group: start a conversation with it, + send a message, and read the reply back. "Deployed" only means the config + loaded; a test message is the only thing that proves the LLM call, the tool + wiring and any vault key actually resolve at runtime. After creating or + changing an agent, offer this. - Start a conversation, send ONE representative message, then read the conversation back and quote what the agent actually replied. - Each message you send needs the admin's approval, so make it count: say what