Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion .github/scripts/install-backend-cli.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,19 @@ const backends = {
},
};

// `openai-agents` has no external CLI to install — it talks to a
// remote HTTP endpoint over the wire and the live-backend test stands
// up its own in-process dummy server. Short-circuit before the
// backends-table lookup so the CI matrix can include it without
// special-casing the workflow.
if (backend === "openai-agents") {
console.log("openai-agents needs no external CLI; skipping install.");
process.exit(0);
}

if (!backend || !Object.hasOwn(backends, backend)) {
console.error(
`Usage: node .github/scripts/install-backend-cli.mjs <${Object.keys(backends).join("|")}>`,
`Usage: node .github/scripts/install-backend-cli.mjs <${Object.keys(backends).join("|")}|openai-agents>`,
);
process.exit(2);
}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
backend: [kilo, opencode, claude, codex]
backend: [kilo, opencode, claude, codex, openai-agents]
steps:
- uses: actions/checkout@v6

Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
"test:kilo:backend": "vitest run --reporter=verbose --reporter=json --outputFile=kilo-backend-results.json src/__tests__/integration/kilo-live-discovery.test.ts",
"test:opencode:backend": "vitest run --reporter=verbose --reporter=json --outputFile=opencode-backend-results.json src/__tests__/integration/opencode-live-discovery.test.ts",
"test:codex:backend": "vitest run --reporter=verbose --reporter=json --outputFile=codex-backend-results.json src/__tests__/integration/codex-live-discovery.test.ts",
"test:openai-agents:backend": "vitest run --reporter=verbose --reporter=json --outputFile=openai-agents-backend-results.json src/__tests__/integration/openai-agents-live-discovery.test.ts",
"tarball:check": "node .github/scripts/tarball-check.mjs",
"build:stub-sea": "node src/__tests__/integration/stub-claude/build-sea.mjs",
"test:watch": "vitest",
Expand Down
59 changes: 59 additions & 0 deletions src/__tests__/fs-path.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/**
* Tests for the shared `expandFsPath` helper. The single thing this
* has to get right is that `~/` becomes `$HOME/` — every
* model-supplied path that crosses into `fs.*` or `bot.api.send*`
* goes through this, and Node's `fs` module does NOT expand tildes
* itself. A regression here would resurface bugs like ENOENT on
* `~/.talon/workspace/robot.svg` from `send_file`.
*/
import { describe, it, expect } from "vitest";
import { homedir } from "node:os";
import { resolve, isAbsolute, sep } from "node:path";
import { expandFsPath } from "../util/fs-path.js";

describe("expandFsPath", () => {
it("expands a bare ~ to the home directory", () => {
expect(expandFsPath("~")).toBe(homedir());
});

it("expands ~/<rel> to $HOME/<rel>", () => {
expect(expandFsPath("~/.talon/workspace/robot.svg")).toBe(
resolve(homedir(), ".talon/workspace/robot.svg"),
);
});

it("returns absolute POSIX-style paths unchanged on POSIX, absolute Windows paths unchanged on Windows", () => {
// `path.isAbsolute` accepts `/foo` as absolute on both platforms
// (it's the POSIX shape), but `path.resolve` on Windows will
// prepend the current drive letter — so equality is only safe
// when we compare against an actually-absolute-on-this-platform
// input. Build one from the test's own resolved cwd.
const abs = resolve(process.cwd(), "abs-test-file");
expect(expandFsPath(abs)).toBe(abs);
});

it("resolves relative paths against process.cwd()", () => {
const out = expandFsPath("relative/file.txt");
expect(isAbsolute(out)).toBe(true);
// path.resolve normalises separators to the platform default
// (backslashes on Windows), so use the resolved comparison value
// rather than a hard-coded POSIX suffix.
expect(out).toBe(resolve(process.cwd(), "relative/file.txt"));
});

it("returns an empty string unchanged", () => {
expect(expandFsPath("")).toBe("");
});

it("preserves the leading tilde on `~foo` (NOT a home-relative path)", () => {
// `~foo` is NOT a home-relative path (that would be `~/foo`) —
// it's a literal filename starting with a tilde. We must NOT
// expand it as if the user meant `~/foo`. Resolve as relative;
// the resulting absolute path ends with `<sep>~weird` on both
// POSIX (sep=`/`) and Windows (sep=`\`). If the expander had
// mistakenly treated `~weird` as home-relative the path would
// end with `~weird` directly (without a preceding separator).
const out = expandFsPath("~weird");
expect(out.endsWith(`${sep}~weird`)).toBe(true);
});
});
257 changes: 257 additions & 0 deletions src/__tests__/integration/dummy-openai-server.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,257 @@
/**
* Minimal OpenAI-compatible HTTP server for offline live-backend
* tests against the `openai-agents` backend.
*
* Implements just enough of the surface the SDK actually calls when
* talking to a chat-completions provider:
*
* - `GET /v1/models` — returns a synthetic catalog. The backend's
* `fetchEndpointModels()` reads this on init.
* - `POST /v1/chat/completions` (with `stream: true`) — returns a
* scripted Server-Sent-Events response containing assistant text
* and/or tool calls. The SDK parses these chunks into
* `RunItemStreamEvent`s the same way it does for real providers.
*
* The script is a list of one ScriptedResponse per *expected request*
* from the SDK. A turn that requires the model to call a tool and
* then respond emits two HTTP requests — one whose response asks for
* the tool, and a second (with the tool's output appended to the
* messages) whose response is the final assistant text. Tests
* arrange the script accordingly.
*
* No auth check, no rate limits, no streaming gymnastics — the goal
* is deterministic SSE bytes the SDK can parse, not provider
* fidelity.
*/
import { createServer, type Server } from "node:http";
import { randomBytes } from "node:crypto";
import type { AddressInfo } from "node:net";

export interface DummyModel {
id: string;
name?: string;
context_length?: number;
/** OpenRouter-style: `"0"` flags free-tier. */
pricing?: { prompt: string; completion?: string };
}

export interface ToolCallSpec {
name: string;
/** Arguments — serialised to a JSON string by the server (matches OpenAI's API). */
arguments: Record<string, unknown>;
/** Optional callId; auto-generated when omitted. */
id?: string;
}

/**
* One response in the per-request script. `text` and `toolCalls` may
* both be present (an assistant chunk that also requests tools); when
* `toolCalls` is set, `finishReason` defaults to `"tool_calls"`.
*/
export interface ScriptedResponse {
text?: string;
toolCalls?: ToolCallSpec[];
finishReason?: "stop" | "tool_calls";
/** Optional model id to echo in the response (defaults to the request's). */
model?: string;
}

export interface RecordedRequest {
path: string;
method: string;
body: unknown;
}

export interface DummyOpenAIServer {
url: string;
port: number;
close(): Promise<void>;
/** Set the response sequence for upcoming chat-completions requests. Consumed in order. */
setScript(responses: ScriptedResponse[]): void;
/** Empty the pending script (does NOT touch the recorded-request log). */
clearScript(): void;
/** Inspect requests recorded since last `clearRequests()`. */
getRequests(): RecordedRequest[];
clearRequests(): void;
}

export interface DummyServerOptions {
models?: DummyModel[];
}

const DEFAULT_MODELS: DummyModel[] = [
{
id: "test/gpt-stub",
name: "GPT Stub",
context_length: 8192,
pricing: { prompt: "0" },
},
];

export async function startDummyOpenAIServer(
options: DummyServerOptions = {},
): Promise<DummyOpenAIServer> {
const models = options.models ?? DEFAULT_MODELS;
let script: ScriptedResponse[] = [];
const requests: RecordedRequest[] = [];

const server: Server = createServer(async (req, res) => {
const url = req.url ?? "/";
let body = "";
req.on("data", (chunk) => {
body += chunk;
});
req.on("end", () => {
let parsed: unknown = undefined;
if (body) {
try {
parsed = JSON.parse(body);
} catch {
parsed = body;
}
}
requests.push({ path: url, method: req.method ?? "GET", body: parsed });

if (url.endsWith("/models") && req.method === "GET") {
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({ data: models }));
return;
}

if (url.endsWith("/chat/completions") && req.method === "POST") {
const reqBody = parsed as { model?: string } | undefined;
const next = script.shift();
if (!next) {
res.writeHead(500, { "Content-Type": "application/json" });
res.end(
JSON.stringify({
error: {
message:
"dummy server: no scripted response available for this request",
code: "no_script",
},
}),
);
return;
}
writeChatCompletionSSE(
res,
next,
reqBody?.model ?? models[0]?.id ?? "test/gpt-stub",
);
return;
}

// Unknown path — return 404 so the SDK surfaces a useful error.
res.writeHead(404, { "Content-Type": "application/json" });
res.end(JSON.stringify({ error: { message: `unknown path: ${url}` } }));
});
});

await new Promise<void>((resolve) => {
server.listen(0, "127.0.0.1", resolve);
});

const port = (server.address() as AddressInfo).port;
const url = `http://127.0.0.1:${port}/v1`;

return {
url,
port,
close: () => new Promise<void>((resolve) => server.close(() => resolve())),
setScript: (responses) => {
script = [...responses];
},
clearScript: () => {
script = [];
},
getRequests: () => requests.slice(),
clearRequests: () => {
requests.length = 0;
},
};
}

/**
* Stream a single chat-completions response over Server-Sent Events.
* The format matches OpenAI's chat-completions streaming protocol so
* the official `openai` client (used by `@openai/agents` under the
* hood) parses it without modification.
*/
function writeChatCompletionSSE(
res: import("node:http").ServerResponse,
scripted: ScriptedResponse,
modelId: string,
): void {
res.writeHead(200, {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache",
Connection: "keep-alive",
});

const id = `chatcmpl-${randomBytes(8).toString("hex")}`;
const created = Math.floor(Date.now() / 1000);
const model = scripted.model ?? modelId;
const finish =
scripted.finishReason ??
(scripted.toolCalls && scripted.toolCalls.length > 0
? "tool_calls"
: "stop");

const writeChunk = (delta: Record<string, unknown>): void => {
res.write(
`data: ${JSON.stringify({
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta, finish_reason: null }],
})}\n\n`,
);
};

// Opening role chunk — mirrors what real providers emit first.
writeChunk({ role: "assistant", content: "" });

// Text content streamed as one chunk for determinism. Real providers
// chunk word-by-word; the SDK doesn't care.
if (scripted.text) {
writeChunk({ content: scripted.text });
}

// Tool calls. Each call gets a single chunk carrying name + the
// fully-serialised arguments (real providers stream arguments
// character-by-character; emitting the whole blob once is a valid
// shape the parser handles).
if (scripted.toolCalls && scripted.toolCalls.length > 0) {
const toolCalls = scripted.toolCalls.map((tc, i) => ({
index: i,
id: tc.id ?? `call_${randomBytes(8).toString("hex")}`,
type: "function" as const,
function: {
name: tc.name,
arguments: JSON.stringify(tc.arguments),
},
}));
writeChunk({ tool_calls: toolCalls });
}

// Final chunk with finish_reason — terminates the SSE stream from
// the SDK's perspective.
res.write(
`data: ${JSON.stringify({
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: {}, finish_reason: finish }],
usage: {
prompt_tokens: 10,
completion_tokens: scripted.text ? scripted.text.length : 0,
total_tokens: 10 + (scripted.text ? scripted.text.length : 0),
},
})}\n\n`,
);
res.write("data: [DONE]\n\n");
res.end();
}
Loading
Loading