Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion server/drivers/acp/acp.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -225,10 +225,10 @@ describe("ACP turns (fake CLI)", () => {
"turn.started",
"session.started",
"content.delta",
"item.completed", // assistant_text before the tool, not summed on settle
"item.started", // tool tc-1
"item.completed", // tool tc-1 done
"thread.token-usage.updated",
"item.completed", // assistant_text (summed) on settle
"turn.completed",
]);
expect(recorder.events.every((e) => e.turnId === turnId && e.provider === "grokAgent")).toBe(true);
Expand All @@ -241,6 +241,34 @@ describe("ACP turns (fake CLI)", () => {
expect(instance.adapter.hasSession("t-happy")).toBe(false);
});

it("emits each assistant text block before the tool that follows it", async () => {
await create(GrokAgentDriver, "interleave");
await instance.adapter.sendTurn({ threadId: "t-interleave", text: "go", model: "grok-4.5" });
await recorder.until((e) => e.type === "turn.completed");

const types = recorder.events.map((e) => e.type);
expect(types).toEqual([
"turn.started",
"session.started",
"content.delta",
"item.completed", // before one
"item.started", // tc-1
"item.completed", // tc-1
"content.delta",
"item.completed", // before two
"item.started", // tc-2
"item.completed", // tc-2
"content.delta",
"thread.token-usage.updated",
"item.completed", // after — no following tool, so settle flushes
"turn.completed",
]);
const texts = recorder.events
.filter((e) => e.type === "item.completed" && (e as { itemType?: string }).itemType === "assistant_text")
.map((e) => (e as { text: string }).text);
expect(texts).toEqual(["before one", "before two", "after"]);
});

it("reads token usage from the root of the prompt result", async () => {
process.env.FAKE_ACP_USAGE_ROOT = "1";
await create();
Expand Down
14 changes: 11 additions & 3 deletions server/drivers/acp/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,14 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver<AcpConfig>

const stop = () => killCliTree(child);

/** Emit buffered assistant text as its own item, then clear it. */
const flushAssistantText = () => {
const text = state.text;
if (!text.trim()) return;
state.text = "";
emit({ ...base(threadId, turnId), type: "item.completed", itemType: "assistant_text", text });
};

const settle = (ok: boolean, stopReason: string | null) => {
if (state.settled) return;
state.settled = true;
Expand All @@ -328,9 +336,7 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver<AcpConfig>
}
rpcPending.clear();
active.delete(threadId);
if (state.text.trim()) {
emit({ ...base(threadId, turnId), type: "item.completed", itemType: "assistant_text", text: state.text });
}
flushAssistantText();
emit({ ...base(threadId, turnId), type: "turn.completed", ok, stopReason, cost: null });
stop(); // the agent process does not exit on its own
};
Expand All @@ -342,6 +348,7 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver<AcpConfig>
return send({ jsonrpc: "2.0", id: msg.id, error: { code: -32601, message: "method not found" } });
}
const params = msg.params ?? {};
flushAssistantText();
const options: Array<{ optionId?: string; kind?: string }> = Array.isArray(params.options) ? params.options : [];
const optionFor = (want: "allow" | "reject") =>
options.find((o) => String(o.kind ?? "").startsWith(want) && typeof o.optionId === "string")?.optionId ?? null;
Expand Down Expand Up @@ -428,6 +435,7 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver<AcpConfig>
break;
}
case "tool_call": {
flushAssistantText();
emit({
...base(threadId, turnId),
type: "item.started",
Expand Down
200 changes: 200 additions & 0 deletions server/drivers/boxagent.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
// Box agent contract tests against a scripted fake of ascii.dev's box HTTP
// API. The driver polls events + prompt status; the fake advances one poll
// per GET so we can assert message → tool → message order without sleeping.
import { afterEach, beforeEach, describe, expect, it } from "vitest";

import { ensureDirs } from "../config.ts";
import type { ProviderInstance } from "../contracts.ts";
import { recordEvents, type EventRecorder } from "../testing/events.ts";
import { BoxAgentDriver } from "./boxagent.ts";

const BOX = "box-1";
const PROMPT = "p1";

/** JSON Response helper for the in-process Box HTTP fake. */
function json(body: unknown, status = 200) {
return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
}

type Poll = { events: unknown[]; status?: { promptRun: { status: string; result?: string } } };

/** Stub fetch so each GET /events + /prompts pair advances one poll in `script`. */
function installFakeBox(script: Poll[]) {
let i = 0;
const previous = globalThis.fetch;
globalThis.fetch = (async (input: string | URL, init?: RequestInit) => {
const url = String(input);
const method = String(init?.method ?? "GET").toUpperCase();
if (url.endsWith("/me")) return json({ ok: true });
if (method === "POST" && /\/boxes\/[^/]+\/prompt$/.test(url)) return json({ promptRun: { id: PROMPT } });
if (method === "POST" && url.includes("/interrupt")) return json({ ok: true });
if (url.includes("/events")) {
const step = script[Math.min(i, script.length - 1)]!;
i += 1;
return json({ events: step.events });
}
if (url.includes(`/prompts/${PROMPT}`)) {
const step = script[Math.min(Math.max(i - 1, 0), script.length - 1)]!;
return json(step.status ?? { promptRun: { status: "running" } });
}
return json({ error: `unexpected ${method} ${url}` }, 404);
}) as typeof fetch;
return () => {
globalThis.fetch = previous;
};
}

const computer = { boxId: BOX, token: "box-test-token" };

describe("BoxAgentDriver turns (fake API)", () => {
let instance: ProviderInstance;
let recorder: EventRecorder;
let restoreFetch: (() => void) | undefined;

const create = async () => {
instance = await BoxAgentDriver.create({
instanceId: "box-test",
displayName: "Box Test",
environment: { BOX_TOKEN: "box-test-token" },
enabled: true,
config: { pollMs: 0 },
});
recorder = recordEvents(instance.adapter);
};

beforeEach(() => {
ensureDirs();
});

afterEach(async () => {
recorder?.stop();
await instance?.dispose();
restoreFetch?.();
restoreFetch = undefined;
});

it("flushes prefix-grown text before a tool, then the tail at settle", async () => {
restoreFetch = installFakeBox([
{
events: [{ id: "e1", type: "response", text: "hel" }],
status: { promptRun: { status: "running" } },
},
{
events: [
{ id: "e1", type: "response", text: "hel" },
{ id: "e2", type: "tool", title: "run" },
],
status: { promptRun: { status: "running" } },
},
{
events: [
{ id: "e1", type: "response", text: "hel" },
{ id: "e2", type: "tool", title: "run" },
{ id: "e3", type: "response", text: "hello there" },
],
status: { promptRun: { status: "finished", result: "hello there" } },
},
]);
await create();
await instance.adapter.sendTurn({ threadId: "t-prefix", text: "go", integrations: { computer } });
await recorder.until((e) => e.type === "turn.completed");

const texts = recorder.events
.filter((e) => e.type === "item.completed" && (e as { itemType: string }).itemType === "assistant_text")
.map((e) => (e as { text: string }).text);
expect(texts).toEqual(["hel", "lo there"]);
});

it("keeps a non-prefix response after a flush instead of slicing it away", async () => {
restoreFetch = installFakeBox([
{
events: [{ id: "e1", type: "response", text: "before" }],
status: { promptRun: { status: "running" } },
},
{
events: [
{ id: "e1", type: "response", text: "before" },
{ id: "e2", type: "tool", title: "run" },
],
status: { promptRun: { status: "running" } },
},
{
events: [
{ id: "e1", type: "response", text: "before" },
{ id: "e2", type: "tool", title: "run" },
{ id: "e3", type: "response", text: "after" },
],
status: { promptRun: { status: "finished", result: "after" } },
},
]);
await create();
await instance.adapter.sendTurn({ threadId: "t-nonprefix", text: "go", integrations: { computer } });
await recorder.until((e) => e.type === "turn.completed");

const types = recorder.events.map((e) => e.type);
expect(types).toEqual([
"turn.started",
"session.started",
"content.delta",
"item.completed", // before
"item.started",
"content.delta",
"item.completed", // after — must not be sliced to ""
"turn.completed",
]);
const texts = recorder.events
.filter((e) => e.type === "item.completed" && (e as { itemType: string }).itemType === "assistant_text")
.map((e) => (e as { text: string }).text);
expect(texts).toEqual(["before", "after"]);
});

it("ingests a non-prefix prompt result when events already set lastText", async () => {
restoreFetch = installFakeBox([
{
events: [{ id: "e1", type: "response", text: "before" }],
status: { promptRun: { status: "running" } },
},
{
events: [
{ id: "e1", type: "response", text: "before" },
{ id: "e2", type: "tool", title: "run" },
],
status: { promptRun: { status: "running" } },
},
{
events: [
{ id: "e1", type: "response", text: "before" },
{ id: "e2", type: "tool", title: "run" },
],
status: { promptRun: { status: "finished", result: "done" } },
},
]);
await create();
await instance.adapter.sendTurn({ threadId: "t-status", text: "go", integrations: { computer } });
await recorder.until((e) => e.type === "turn.completed");

const texts = recorder.events
.filter((e) => e.type === "item.completed" && (e as { itemType: string }).itemType === "assistant_text")
.map((e) => (e as { text: string }).text);
expect(texts).toEqual(["before", "done"]);
});

it("flushes pending assistant text when the turn is interrupted", async () => {
restoreFetch = installFakeBox([
{
events: [{ id: "e1", type: "response", text: "half" }],
status: { promptRun: { status: "running" } },
},
]);
await create();
await instance.adapter.sendTurn({ threadId: "t-cancel", text: "go", integrations: { computer } });
await recorder.until((e) => e.type === "content.delta");
await instance.adapter.interruptTurn("t-cancel");
const done = await recorder.until((e) => e.type === "turn.completed");
expect(done).toMatchObject({ ok: false, stopReason: "interrupted" });
const texts = recorder.events
.filter((e) => e.type === "item.completed" && (e as { itemType: string }).itemType === "assistant_text")
.map((e) => (e as { text: string }).text);
expect(texts).toEqual(["half"]);
});
});
52 changes: 26 additions & 26 deletions server/drivers/boxagent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,22 @@ export const BoxAgentDriver: ProviderDriver<BoxAgentConfig> = {
const seen = new Set<string>();
const startedAt = Date.now();
let lastText = "";
let pendingText = "";
/** Emit unflushed deltas as assistant_text and reset pendingText. */
const flushAssistantText = () => {
const text = pendingText;
pendingText = "";
if (!text.trim()) return;
emit({ ...base(threadId, turnId), type: "item.completed", itemType: "assistant_text", text });
};
/** Stream a full-text snapshot as a delta and accumulate it for flush. */
const ingest = (text: string) => {
const delta = text.startsWith(lastText) ? text.slice(lastText.length) : text;
lastText = text;
if (!delta) return;
pendingText += delta;
emit({ ...base(threadId, turnId), type: "content.delta", streamKind: "assistant_text", delta });
};
try {
for (;;) {
if (cancelled) break;
Expand All @@ -148,12 +164,9 @@ export const BoxAgentDriver: ProviderDriver<BoxAgentConfig> = {
// stream anyway.
const text = ev.text ?? ev.message ?? ev.data?.text ?? ev.data?.content ?? null;
if (/assistant|message|output|response/i.test(kind) && typeof text === "string" && text.trim()) {
const delta = text.startsWith(lastText) ? text.slice(lastText.length) : text;
lastText = text;
if (delta) {
emit({ ...base(threadId, turnId), type: "content.delta", streamKind: "assistant_text", delta });
}
ingest(text);
} else if (/tool|command|exec|browse/i.test(kind)) {
flushAssistantText();
emit({
...base(threadId, turnId),
type: "item.started",
Expand All @@ -167,9 +180,7 @@ export const BoxAgentDriver: ProviderDriver<BoxAgentConfig> = {
// events themselves instead of hanging to the 30-min ceiling
if (!promptId && /complete|finish|done|success|fail|error/i.test(kind)) {
active.delete(threadId);
if (lastText) {
emit({ ...base(threadId, turnId), type: "item.completed", itemType: "assistant_text", text: lastText });
}
flushAssistantText();
const failed = /fail|error/i.test(kind);
emit({ ...base(threadId, turnId), type: "turn.completed", ok: !failed, stopReason: failed ? kind : null, cost: null });
return;
Expand All @@ -184,30 +195,17 @@ export const BoxAgentDriver: ProviderDriver<BoxAgentConfig> = {
const state = String(run?.status ?? "");
if (/completed|succeeded|done|finished/i.test(state)) {
const result = run?.result ?? run?.output ?? lastText;
// stream only the growth past what events already sent —
// the settled message below carries the full text regardless
if (typeof result === "string" && result.trim() && result !== lastText && result.startsWith(lastText)) {
emit({
...base(threadId, turnId),
type: "content.delta",
streamKind: "assistant_text",
delta: result.slice(lastText.length),
});
if (typeof result === "string" && result.trim() && result !== lastText) {
ingest(result);
}
emit({
...base(threadId, turnId),
type: "item.completed",
itemType: "assistant_text",
text: typeof result === "string" && result.trim() ? result : lastText || "(finished)",
});
if (!pendingText.trim() && !lastText.trim()) pendingText = "(finished)";
flushAssistantText();
active.delete(threadId);
emit({ ...base(threadId, turnId), type: "turn.completed", ok: true, stopReason: null, cost: null });
return;
}
if (/failed|error|cancelled|interrupted/i.test(state)) {
if (lastText) {
emit({ ...base(threadId, turnId), type: "item.completed", itemType: "assistant_text", text: lastText });
}
flushAssistantText();
Comment thread
coderabbitai[bot] marked this conversation as resolved.
active.delete(threadId);
emit({ ...base(threadId, turnId), type: "turn.completed", ok: false, stopReason: state, cost: null });
return;
Expand All @@ -218,9 +216,11 @@ export const BoxAgentDriver: ProviderDriver<BoxAgentConfig> = {
}
}
// cancelled
flushAssistantText();
active.delete(threadId);
emit({ ...base(threadId, turnId), type: "turn.completed", ok: false, stopReason: "interrupted", cost: null });
} catch (e) {
flushAssistantText();
active.delete(threadId);
emit({ ...base(threadId, turnId), type: "runtime.error", message: (e as Error).message });
emit({ ...base(threadId, turnId), type: "turn.completed", ok: false, stopReason: "error", cost: null });
Expand Down
Loading