feat(operator): let it test-drive an agent or group by actually talking to it

src/lib/operator/__tests__/system-prompt.test.ts

@@ -6,7 +6,13 @@ import {
   defaultOperatorPromptBody,
   safetyPreambleForScope,
 } from "../system-prompt";
-import { READ_ENDPOINTS, WRITE_ENDPOINTS, endpointsForScope } from "../tool-scopes";
+import {
+  READ_ENDPOINTS,
+  WRITE_ENDPOINTS,
+  endpointsForScope,
+  buildToolApprovals,
+  grantsConversationTesting,
+} from "../tool-scopes";
 
 /**
  * A granted set that contains a write.
@@ -397,3 +403,106 @@ describe("prompt corrections from dev testing", () => {
     expect(defaultOperatorPromptBody("read_only")).not.toContain("setupAgent essentials");
   });
 });
+
+/**
+ * Test-drive: the operator could build an agent but never exercise one. Asked to
+ * check its own creation it answered "I don't have a start conversation tool" —
+ * accurate, and useless to the admin who had just approved the build.
+ */
+describe("test-drive: talking to another agent", () => {
+  it("grants start + say to read_write, and the read-back to both", () => {
+    const write = new Set(endpointsForScope("read_write"));
+    expect(write.has("POST /agents/{agentId}/start")).toBe(true);
+    expect(write.has("POST /agents/{conversationId}")).toBe(true);
+    expect(write.has("POST /groups/{groupId}/conversations")).toBe(true);
+
+    // The POSTs are writes by method, so they sit in WRITE_ENDPOINTS: putting
+    // them in READ_ENDPOINTS flipped read_only into the write branch of the
+    // safety preamble. Reading a conversation back is a plain GET and is
+    // granted to both scopes.
+    for (const scope of ["read_only", "read_write"] as const) {
+      expect(new Set(endpointsForScope(scope)).has("GET /agents/{conversationId}")).toBe(true);
+    }
+    const read = new Set(endpointsForScope("read_only"));
+    expect(read.has("POST /agents/{agentId}/start")).toBe(false);
+    expect(read.has("POST /agents/{conversationId}")).toBe(false);
+  });
+
+  it("leaves read_only genuinely read-only — the regression the tests caught", () => {
+    expect(safetyPreambleForScope("read_only")).toContain("You are read-only");
+  });
+
+  /**
+   * The prompt must never describe a capability the agent lacks — the whole
+   * reason this module derives from the endpoint set. A first attempt put the
+   * test-drive bullet in BODY_ROLE, which is unconditional, so a read_only
+   * operator was told to start conversations it has no endpoint for. Asserting
+   * on the section heading alone would NOT have caught that, so this checks
+   * every phrase that promises the capability.
+   */
+  it("says nothing whatsoever about test-driving in a read_only body", () => {
+    const body = defaultOperatorPromptBody("read_only");
+    for (const promise of [
+      "Testing an agent",
+      "TEST-DRIVE",
+      "Start a conversation",
+      "start a conversation with it",
+    ]) {
+      expect(body).not.toContain(promise);
+    }
+  });
+
+  /**
+   * THE regression guard. `/resume` would let the operator approve its own
+   * pauses — a complete escape from the gate — and the other three let it
+   * rewrite or discard a conversation's lifecycle. All are excluded by
+   * planning/operator-write-scope-plan.md §5.
+   */
+  it("never grants resume, state, cancel or end — for any scope", () => {
+    for (const scope of ["read_only", "read_write"] as const) {
+      const set = new Set(endpointsForScope(scope));
+      for (const forbidden of [
+        "POST /agents/{conversationId}/resume",
+        "PATCH /agents/{conversationId}/state",
+        "POST /agents/{conversationId}/cancel",
+        "POST /agents/{conversationId}/endConversation",
+        "POST /agents/{conversationId}/undo",
+        "POST /agents/{conversationId}/redo",
+      ]) {
+        expect(set.has(forbidden)).toBe(false);
+      }
+    }
+  });
+
+  it("keeps the gate intact — the new POSTs are approved, never exempt", () => {
+    // Adding a conversation POST to `exempt` would be the first hole ever
+    // punched in http.post:*, and verifyGateInstalled would reject it anyway.
+    expect(buildToolApprovals().exempt).toEqual(["http.get:*"]);
+    expect(buildToolApprovals().requireApproval).toContain("http.post:*");
+  });
+
+  it("tells the operator that an AWAITING_HUMAN reply is a PASS, not a failure", () => {
+    // An agent with its own gate is supposed to stop. Reading that as broken
+    // would report a correctly-configured agent as failing.
+    const body = defaultOperatorPromptBody("read_write");
+    expect(body).toContain("Testing an agent");
+    expect(body).toMatch(/paused on ITS OWN approval gate/);
+    expect(body).toMatch(/That is a PASS/);
+    expect(body).toMatch(/cannot approve on another\s+agent's behalf/);
+  });
+
+  it("says nothing about test-driving when the endpoints are not granted", () => {
+    // The module's rule: the prompt may never describe a capability the agent
+    // lacks. Pass a set with the reads but neither conversation POST.
+    const body = buildOperatorPromptBody(["GET /agentstore/agents/descriptors"]);
+    expect(body).not.toContain("Testing an agent");
+  });
+
+  it("requires BOTH start and say — start alone proves nothing", () => {
+    expect(grantsConversationTesting(["POST /agents/{agentId}/start"])).toBe(false);
+    expect(grantsConversationTesting(["POST /agents/{conversationId}"])).toBe(false);
+    expect(
+      grantsConversationTesting(["POST /agents/{agentId}/start", "POST /agents/{conversationId}"]),
+    ).toBe(true);
+  });
+});

src/lib/operator/__tests__/tool-scopes.test.ts

@@ -94,6 +94,15 @@ describe("tool-scopes", () => {
       // addition here is exactly as dangerous as a silent removal from an
       // allow-list — this test catches either direction.
       expect(WRITE_ENDPOINTS).toEqual([
+        // Test-drive. Writes by METHOD only — they create a conversation and a
+        // message, never a configuration change. Here rather than in
+        // READ_ENDPOINTS because grantsWriteCapability keys off the method, and
+        // a POST among the reads flipped read_only into the write branch of the
+        // safety preamble. `/resume`, `/state`, `/cancel` and `/endConversation`
+        // are excluded on purpose — see TEST_DRIVE_WRITES.
+        "POST /agents/{agentId}/start",
+        "POST /agents/{conversationId}",
+        "POST /groups/{groupId}/conversations",
         "PATCH /descriptorstore/descriptors/{id}",
         "POST /administration/{environment}/deploy/{agentId}",
         "POST /administration/{environment}/undeploy/{agentId}",

src/lib/operator/system-prompt.ts

@@ -3,6 +3,7 @@ import {
   grantsWriteCapability,
   grantsAgentCreation,
   grantsAgentModification,
+  grantsConversationTesting,
   type OperatorScope,
 } from "./tool-scopes";
 import { MODEL_SUGGESTIONS } from "@/lib/model-suggestions";
@@ -127,6 +128,38 @@ You can:
   not exist here — then read the ones you need. Prefer citing the docs over
   answering "how does EDDI do X?" from memory.`;
 
+/**
+ * How to use the runtime conversation endpoints well.
+ *
+ * Included whenever those endpoints are granted, in BOTH scopes — a read-only
+ * operator can still be asked "is this agent actually answering?", and that is a
+ * diagnosis, not a change.
+ *
+ * The `AWAITING_HUMAN` rule is the one that matters: an agent with its own
+ * approval gate is SUPPOSED to stop, and an operator that reads that as a
+ * failure would report a correctly-configured agent as broken.
+ */
+const BODY_TEST_DRIVE = `Testing an agent (or a group) by talking to it:
+- You can TEST-DRIVE any deployed agent or group: start a conversation with it,
+  send a message, and read the reply back. "Deployed" only means the config
+  loaded; a test message is the only thing that proves the LLM call, the tool
+  wiring and any vault key actually resolve at runtime. After creating or
+  changing an agent, offer this.
+- Start a conversation, send ONE representative message, then read the
+  conversation back and quote what the agent actually replied.
+- Each message you send needs the admin's approval, so make it count: say what
+  you are about to send and why, and prefer one good test message to a
+  conversation.
+- Use the agent's own environment. An agent deployed to \`test\` is not
+  reachable in \`production\`, and "no response" from the wrong environment is a
+  false alarm — check deployment status first if you are unsure.
+- If the conversation comes back \`AWAITING_HUMAN\`, the agent you are testing
+  paused on ITS OWN approval gate. That is a PASS, not a failure: it proves the
+  gate works. Report which call it stopped on. You cannot approve on another
+  agent's behalf — say so and let the admin decide.
+- If the reply is empty or an error, report it verbatim with the conversation id
+  rather than re-sending. A failing agent is a finding, not something to retry.`;
+
 /**
  * Architecture background, present in BOTH scopes — a read-only operator
  * diagnosing "my change did nothing" needs the versioning model exactly as
@@ -396,6 +429,9 @@ export function buildOperatorPromptBody(endpoints: readonly string[]): string {
     BODY_ROLE,
     BODY_ARCHITECTURE,
     BODY_CHEATSHEET,
+    // Gated on the endpoints actually granted, not on scope: this module's rule
+    // is that the prompt can never describe a capability the agent lacks.
+    ...(grantsConversationTesting(endpoints) ? [BODY_TEST_DRIVE] : []),
     BODY_APP_CONTEXT,
     buildModelCatalogueSection(),
     BODY_STYLE,

src/lib/operator/tool-scopes.ts

@@ -116,6 +116,11 @@ export const READ_ENDPOINTS: readonly string[] = [
   // Conversations
   "GET /conversationstore/conversations",
   "GET /conversationstore/conversations/{conversationId}",
+  // Test-drive, read half: inspect a conversation with another agent. The POSTs
+  // that CREATE one live in WRITE_ENDPOINTS — see TEST_DRIVE_WRITES.
+  "GET /agents/{conversationId}",
+  "GET /agents/{conversationId}/status",
+  "GET /groups/{groupId}/conversations/{groupConversationId}",
   // Operations
   "GET /administration/{environment}/deploymentstatus/{agentId}",
   "GET /administration/coordinator/status",
@@ -251,7 +256,38 @@ export const READ_ENDPOINTS: readonly string[] = [
  * unconditionally, anything added here is gated the moment it is added — the
  * failure mode of forgetting to update a pattern list does not exist.
  */
+/**
+ * Starting a conversation with another agent, and saying something into it.
+ *
+ * These are POSTs, so they belong in the write set — not because they change any
+ * configuration (they do not), but because `grantsWriteCapability` and the whole
+ * gate model key off the method. Putting them in `READ_ENDPOINTS` flipped a
+ * read-only operator into the write branch of the safety preamble: it lost the
+ * "you are read-only" rule and gained the six write rules, describing a
+ * capability boundary it was not actually behind. The tests caught it; the
+ * lesson is that `READ_ENDPOINTS` means "GETs", not "harmless".
+ *
+ * The consequence, accepted deliberately: a read-only operator cannot test-drive
+ * an agent. That is coherent — it cannot create the conversation record either.
+ *
+ * Every call here pauses for human approval like any other write. That is the
+ * point: sending a message as the admin is a decision they should see, and the
+ * agent being tested keeps its OWN gate for anything it then does.
+ *
+ * Deliberately ABSENT, and they must stay absent: `POST /agents/{id}/resume`
+ * (the operator approving its own pauses would be a complete escape from the
+ * gate), `PATCH /agents/{conversationId}/state`, `/cancel`, `/endConversation`,
+ * `/undo`, `/redo`, and `DELETE …/conversations/{id}` — see
+ * `planning/operator-write-scope-plan.md` §5 in the EDDI repo.
+ */
+const TEST_DRIVE_WRITES = [
+  "POST /agents/{agentId}/start",
+  "POST /agents/{conversationId}",
+  "POST /groups/{groupId}/conversations",
+] as const;
+
 export const WRITE_ENDPOINTS: readonly string[] = [
+  ...TEST_DRIVE_WRITES,
   "PATCH /descriptorstore/descriptors/{id}",
   "POST /administration/{environment}/deploy/{agentId}",
   "POST /administration/{environment}/undeploy/{agentId}",
@@ -394,6 +430,19 @@ export function grantsAgentCreation(endpoints: readonly string[]): boolean {
   return set.has("POST /administration/agents/setup") || set.has("POST /administration/agents/setup-api");
 }
 
+/**
+ * Whether the granted endpoints let the operator hold a conversation with
+ * another agent — start one AND say something into it.
+ *
+ * Both are required: `start` alone creates an empty conversation and proves
+ * nothing, which is exactly the useless half-capability the prompt must not
+ * advertise.
+ */
+export function grantsConversationTesting(endpoints: readonly string[]): boolean {
+  const set = new Set(endpoints);
+  return set.has("POST /agents/{agentId}/start") && set.has("POST /agents/{conversationId}");
+}
+
 /**
  * Whether the granted endpoints can change an existing agent's behavior,
  * outputs, tool wiring, or pipeline — any workflow or writable