Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions server/contracts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,24 @@ export class ProviderError extends Error {
}
}

/** Reasoning-effort levels, ascending. A union of everything any engine
* accepts; each driver declares the subset its CLI will take. */
export const EFFORT_LEVELS = ["none", "low", "medium", "high", "xhigh", "max"] as const;
export type EffortLevel = (typeof EFFORT_LEVELS)[number];

/** Narrow untrusted API/config input before it becomes a model selection. */
export function isEffortLevel(value: unknown): value is EffortLevel {
return typeof value === "string" && (EFFORT_LEVELS as readonly string[]).includes(value);
}

// ── model selection ────────────────────────────────────────────────────
// "Which model" is a data value carried on the request, never a service
// binding (upstream ModelSelectionWire). instanceId is the routing key.
export interface ModelSelection {
instanceId: InstanceId;
model: string;
/** Optional: no effort means no flag, and the CLI keeps its own default. */
effort?: EffortLevel;
}

// ── instance configuration envelope ────────────────────────────────────
Expand Down Expand Up @@ -110,6 +122,7 @@ export interface SendTurnInput {
threadId: ThreadId;
text: string;
model?: string;
effort?: EffortLevel;
resumeCursor?: unknown;
/** Prior turns for transcript-replay providers (API-backed drivers). */
transcript?: Array<{ role: "user" | "assistant"; text: string }>;
Expand Down Expand Up @@ -154,6 +167,10 @@ export interface ProviderAdapter {
* connected apps). Same rule again: a key in the config says the user
* HAS those connections, not that this driver can reach them. */
composioMcp?: boolean;
/** Effort levels this driver can pass to its CLI, ascending. Absent =
* the driver cannot set effort, so the app never offers the control —
* same rule as computerMcp: never show a knob the driver cannot turn. */
effortLevels?: readonly EffortLevel[];
};
sendTurn(input: SendTurnInput): Promise<TurnStartResult>;
interruptTurn(threadId: ThreadId, turnId?: TurnId): Promise<void>;
Expand Down
31 changes: 31 additions & 0 deletions server/drivers/acp/acp.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,37 @@ describe("ACP turns (fake CLI)", () => {

expect(JSON.parse(readFileSync(dump, "utf8")).env.TEST_POLICY).toBe("auto");
});

it("declares effort levels for Grok only", async () => {
await create(GrokAgentDriver);
expect(instance.adapter.capabilities.effortLevels).toEqual(["low", "medium", "high"]);

await create(GeminiAgentDriver);
expect(instance.adapter.capabilities.effortLevels).toBeUndefined();

await create(KimiAgentDriver);
expect(instance.adapter.capabilities.effortLevels).toBeUndefined();
});

it("passes effort to Grok, and omits the flag when unset", async () => {
const withEffort = join(scratch, "grok-effort.json");
await create(GrokAgentDriver);
process.env.FAKE_ACP_DUMP = withEffort;
await instance.adapter.sendTurn({ threadId: "t-effort", text: "hi", effort: "high" });
await recorder.until((e) => e.type === "turn.completed");

const seen = JSON.parse(readFileSync(withEffort, "utf8"));
expect(seen.argv).toContain("--reasoning-effort");
expect(seen.argv[seen.argv.indexOf("--reasoning-effort") + 1]).toBe("high");

const without = join(scratch, "grok-no-effort.json");
await create(GrokAgentDriver);
process.env.FAKE_ACP_DUMP = without;
await instance.adapter.sendTurn({ threadId: "t-no-effort", text: "hi" });
await recorder.until((e) => e.type === "turn.completed");

expect(JSON.parse(readFileSync(without, "utf8")).argv).not.toContain("--reasoning-effort");
});
});

describe("ACP snapshot", () => {
Expand Down
13 changes: 12 additions & 1 deletion server/drivers/acp/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { describeSpawnFailure, execCli, killCliTree, spawnCli } from "../../proc

import type {
DriverCreateInput,
EffortLevel,
EngineInstall,
ProviderDriver,
ProviderInstance,
Expand Down Expand Up @@ -56,6 +57,11 @@ export interface AcpSupport {
driverKind: string;
displayName: string;
models: { default: string; options: Array<{ id: string; label: string }> };
/** Effort levels this harness's CLI accepts, ascending. Omit when it has
* no reasoning-effort control. Static for the same reason `models` is:
* describe() runs before any session exists, so there is no _meta to read
* — eventually both should come from initialize's _meta.modelState. */
effortLevels?: readonly EffortLevel[];
/** Default CLI binary name if the instance config doesn't override it. */
defaultCli: string;
/** Optional live model catalog. A failed lookup keeps the last usable catalog. */
Expand Down Expand Up @@ -639,7 +645,12 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver<AcpConfig>
snapshot,
adapter: {
provider: DRIVER_KIND,
capabilities: { sessionModelSwitch: "unsupported", agentsMcp: true, computerMcp: true },
capabilities: {
sessionModelSwitch: "unsupported",
agentsMcp: true,
computerMcp: true,
effortLevels: support.effortLevels,
},
sendTurn,
interruptTurn: async (threadId) => active.get(threadId)?.interrupt(),
respondToRequest: async (threadId, requestId, decision) => {
Expand Down
7 changes: 7 additions & 0 deletions server/drivers/acp/grok.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ const support: AcpSupport = {
{ id: "grok-4.5", label: "Grok 4.5" },
],
},
// Grok's accepted levels vary by model and the CLI validates lazily — a
// rejected level only logs and falls back. Offer the intersection shared
// by every model in this driver's picker; notably, grok-4.5 rejects xhigh.
effortLevels: ["low", "medium", "high"],
defaultCli: "grok",
nativeSource: "grok.acp",
loginNote: "Grok CLI is not signed in — run `grok login` in a terminal",
Expand All @@ -46,6 +50,9 @@ const support: AcpSupport = {
"--permission-mode",
config.fullAuto ? "bypassPermissions" : "default",
...(turn.model ? ["-m", turn.model] : []),
// long form on purpose: `--effort` is documented as an alias, and an
// alias is the part a CLI is free to rename
...(turn.effort ? ["--reasoning-effort", turn.effort] : []),
"agent",
"stdio",
],
Expand Down
33 changes: 33 additions & 0 deletions server/drivers/claude.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,39 @@ describe("ClaudeDriver turns (fake CLI)", () => {
await instance.adapter.interruptTurn("t-perm-2");
await recorder.until((e) => e.type === "turn.completed");
});

it("passes effort to the CLI, and omits the flag when unset", async () => {
await create();
const dump = join(scratch, "effort.json");
process.env.FAKE_CLAUDE_DUMP = dump;

await instance.adapter.sendTurn({ threadId: "t-effort", text: "hi", effort: "xhigh" });
await recorder.until((e) => e.type === "turn.completed");

const seen = JSON.parse(readFileSync(dump, "utf8"));
expect(seen.argv).toContain("--effort");
expect(seen.argv[seen.argv.indexOf("--effort") + 1]).toBe("xhigh");
expect(seen.argv.filter((a: string) => a === "--effort")).toHaveLength(1);
});

it("adds no effort flag when the turn has none", async () => {
await create();
const dump = join(scratch, "no-effort.json");
process.env.FAKE_CLAUDE_DUMP = dump;

await instance.adapter.sendTurn({ threadId: "t-no-effort", text: "hi" });
await recorder.until((e) => e.type === "turn.completed");

const seen = JSON.parse(readFileSync(dump, "utf8"));
expect(seen.argv).not.toContain("--effort");
});

it("declares the effort levels the CLI accepts", async () => {
await create();
expect(instance.adapter.capabilities.effortLevels).toEqual([
"low", "medium", "high", "xhigh", "max",
]);
});
});

// Auth state must come from the CLI, not from probing its credential store:
Expand Down
9 changes: 8 additions & 1 deletion server/drivers/claude.ts
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ export const ClaudeDriver: ProviderDriver<ClaudeConfig> = {
if (sessionId) args.push("--resume", sessionId);
else args.push("--session-id", newSessionId!);
if (turn.model) args.push("--model", turn.model);
if (turn.effort) args.push("--effort", turn.effort);
if (turn.system) args.push("--append-system-prompt", turn.system);

// integrations → MCP servers; pre-allow their tools (a headless
Expand Down Expand Up @@ -560,7 +561,13 @@ export const ClaudeDriver: ProviderDriver<ClaudeConfig> = {
snapshot,
adapter: {
provider: DRIVER_KIND,
capabilities: { sessionModelSwitch: "in-session", agentsMcp: true, computerMcp: true, composioMcp: true },
capabilities: {
sessionModelSwitch: "in-session",
agentsMcp: true,
computerMcp: true,
composioMcp: true,
effortLevels: ["low", "medium", "high", "xhigh", "max"],
},
sendTurn,
interruptTurn: async (threadId) => active.get(threadId)?.stop(),
respondToRequest: async (threadId, requestId, decision) => {
Expand Down
33 changes: 33 additions & 0 deletions server/drivers/codex.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,4 +197,37 @@ describe("CodexDriver turns (fake app-server)", () => {
expect(done).toMatchObject({ ok: false });
expect(await instance.snapshot()).toMatchObject({ state: "unavailable" });
});

it("declares the effort levels the app-server accepts", async () => {
await create();
expect(instance.adapter.capabilities.effortLevels).toEqual([
"low", "medium", "high", "xhigh", "max",
]);
});

it("sends effort on turn/start, and omits the key when unset", async () => {
await create();
const dump = join(scratch, "effort.json");
process.env.FAKE_CODEX_DUMP = dump;

await instance.adapter.sendTurn({ threadId: "t-effort", text: "hi", effort: "xhigh" });
await recorder.until((e) => e.type === "turn.completed");

const seen = JSON.parse(readFileSync(dump, "utf8"));
const turnStart = seen.calls.find((c: any) => c.method === "turn/start");
expect(turnStart.params.effort).toBe("xhigh");
});

it("sends no effort key when the turn has none", async () => {
await create();
const dump = join(scratch, "no-effort.json");
process.env.FAKE_CODEX_DUMP = dump;

await instance.adapter.sendTurn({ threadId: "t-no-effort", text: "hi" });
await recorder.until((e) => e.type === "turn.completed");

const seen = JSON.parse(readFileSync(dump, "utf8"));
const turnStart = seen.calls.find((c: any) => c.method === "turn/start");
expect(turnStart.params).not.toHaveProperty("effort");
});
});
15 changes: 14 additions & 1 deletion server/drivers/codex.ts
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,16 @@ export const CodexDriver: ProviderDriver<CodexConfig> = {
await request("turn/start", {
threadId: codexThreadId,
input: [{ type: "text", text: turn.system ? `${turn.system}\n\n${turn.text}` : turn.text }],
// Spread, not `effort: turn.effort ?? null`. Probed against
// codex-cli 0.146.0: null is indistinguishable from an absent key
// — both leave the thread's current effort alone, emitting no
// thread/settings/updated, and thread/resume reads the old value
// back. The app-server offers no way to clear a level either:
// "" is rejected outright and thread/start takes no effort at
// all. So a thread keeps the last level it was sent until it is
// sent another, and choosing Default lands on the bot's next new
// thread rather than the current one.
...(turn.effort ? { effort: turn.effort } : {}),
});
} catch (e) {
if (!state.settled) {
Expand Down Expand Up @@ -420,7 +430,10 @@ export const CodexDriver: ProviderDriver<CodexConfig> = {
snapshot,
adapter: {
provider: DRIVER_KIND,
capabilities: { sessionModelSwitch: "unsupported" },
capabilities: {
sessionModelSwitch: "unsupported",
effortLevels: ["low", "medium", "high", "xhigh", "max"],
},
sendTurn,
interruptTurn: async (threadId) => active.get(threadId)?.stop(),
respondToRequest: async (threadId, requestId, decision) => {
Expand Down
18 changes: 18 additions & 0 deletions server/harness/registry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,24 @@ describe("ProviderRegistry", () => {
expect(described.snapshot).toMatchObject({ state: "unavailable", reason: "provider probe exploded" });
});

it("forwards a live instance's declared effort levels in describe()", async () => {
const fake = makeFakeDriver({ effortLevels: ["low", "high"] });
const registry = new ProviderRegistry([fake.driver]);
await registry.load({ a: { driver: "fake" } });

const [described] = await registry.describe();
expect(described.capabilities.effortLevels).toEqual(["low", "high"]);
});

it("omits effortLevels from describe() when the driver declares none", async () => {
const fake = makeFakeDriver();
const registry = new ProviderRegistry([fake.driver]);
await registry.load({ a: { driver: "fake" } });

const [described] = await registry.describe();
expect(described.capabilities.effortLevels).toBeUndefined();
});

it("disposeAll disposes every live instance and empties the registry", async () => {
const fake = makeFakeDriver();
const registry = new ProviderRegistry([fake.driver]);
Expand Down
1 change: 1 addition & 0 deletions server/harness/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ export class ProviderRegistry {
capabilities: {
computerMcp: inst.adapter.capabilities.computerMcp === true,
agentsMcp: inst.adapter.capabilities.agentsMcp === true,
effortLevels: inst.adapter.capabilities.effortLevels,
},
install: this.driversByKind.get(inst.driverKind)?.install,
};
Expand Down
72 changes: 72 additions & 0 deletions server/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,78 @@ describe("harness HTTP API", () => {
}
});

it("keeps the rest of a duplicate's fields when the source engine is offline", async () => {
// duplicateBot POSTs a blank bot, then PATCHes the source's whole
// modelSelection in one body beside its name, title and description.
// "ghost" is an unknown driver, so the registry resolves nothing and the
// level cannot be verified — which must not cost the copy everything
// else in the request.
const copy = (await api("POST", "/api/bots")).body.bot;

const patched = await api("PATCH", `/api/bots/${copy.id}`, {
name: "Reviewer copy",
title: "Reviewer",
description: "reads diffs",
modelSelection: { instanceId: "ghost", model: "ghost-1", effort: "xhigh" },
});

expect(patched.status).toBe(200);
expect(patched.body.bot).toMatchObject({
name: "Reviewer copy",
title: "Reviewer",
description: "reads diffs",
modelSelection: { instanceId: "ghost", model: "ghost-1", effort: "xhigh" },
});
});

it("rejects an unknown effort value even while the engine is offline", async () => {
const bot = (await api("POST", "/api/bots")).body.bot;
const patched = await api("PATCH", `/api/bots/${bot.id}`, {
modelSelection: { instanceId: "ghost", model: "ghost-1", effort: "turbo" },
});

expect(patched.status).toBe(400);
expect(patched.body.error).toContain("not recognized");
});

it("leaves a bot with no effort level untouched", async () => {
const bot = (await api("POST", "/api/bots")).body.bot;
expect(bot.modelSelection.effort).toBeUndefined();

const renamed = await api("PATCH", `/api/bots/${bot.id}`, { name: "Plain" });
expect(renamed.status).toBe(200);
expect(renamed.body.bot.modelSelection.effort).toBeUndefined();
});

// This fixture pins a single unknown driver, so no instance here ever
// resolves: these cover the gate's pass-through and the store's replace
// semantics, NOT the comparison against a live engine's declared list.
// That branch has no coverage at this layer, and manufacturing a live
// instance in this fixture would cost it its no-probe determinism.
it("round-trips an effort level and clears it when the key is dropped", async () => {
const bot = (await api("POST", "/api/bots")).body.bot;
const selection = { instanceId: "ghost", model: "ghost-1" };

const set = await api("PATCH", `/api/bots/${bot.id}`, {
modelSelection: { ...selection, effort: "high" },
});
expect(set.status).toBe(200);
expect(set.body.bot.modelSelection.effort).toBe("high");

const reread = (await api("GET", "/api/bots")).body.bots.find((b: { id: string }) => b.id === bot.id);
expect(reread.modelSelection.effort).toBe("high");

// The panel's "Default" button spreads the selection with effort:
// undefined, and JSON.stringify drops the key — so clearing reaches the
// server as a modelSelection carrying no effort at all.
const cleared = await api("PATCH", `/api/bots/${bot.id}`, { modelSelection: selection });
expect(cleared.status).toBe(200);

const after = (await api("GET", "/api/bots")).body.bots.find((b: { id: string }) => b.id === bot.id);
expect(after.modelSelection).toEqual(selection);
expect(after.modelSelection.effort).toBeUndefined();
});

it("persists an answered onboarding card", async () => {
const { body } = await api("GET", "/api/bots");
const bot = body.bots[0];
Expand Down
Loading
Loading