Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 136 additions & 0 deletions scripts/smoke-ai-counsel.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
// End-to-end smoke test for the ai-counsel driver against a live
// the-ai-counsel backend (jacob-bd/the-ai-counsel, MIT).
//
// Run from the OpenMausBot checkout:
// AI_COUNSEL_URL=http://server.tail85e19.ts.net:8021 \
// node --experimental-strip-types scripts/smoke-ai-counsel.ts
//
// Or run on the homelab server itself (where :8020 is local):
// AI_COUNSEL_URL=http://127.0.0.1:8020 node --experimental-strip-types scripts/smoke-ai-counsel.ts
//
// Note: the smoke uses the local Ollama seat (free, fast) by default so
// it works without spending OpenRouter credits. Override with
// AI_COUNSEL_MODEL=openai/gpt-4.1 to use a real model.
//
// The script:
// 1. constructs the driver with AI_COUNSEL_URL
// 2. asks the counsel for /api/conversations to confirm reachability
// 3. sends a single turn
// 4. prints the per-seat assistant_text events as they arrive
// 5. prints the final synthesis + turn summary
// 6. exits 0 on success, 1 on failure

import { AiCounselDriver } from "../server/drivers/ai-counsel.ts";

const baseUrl = process.env.AI_COUNSEL_URL ?? "http://127.0.0.1:8020";
const model = process.env.AI_COUNSEL_MODEL ?? "ollama:hermes3:8b";
const prompt = process.env.AI_COUNSEL_PROMPT ?? "Reply with exactly: ai-counsel smoke ok";

async function main(): Promise<number> {
console.log(`[smoke] baseUrl=${baseUrl} model=${model}`);

const cfg = AiCounselDriver.decodeConfig({ baseUrl, defaultModel: model });
const instance = await AiCounselDriver.create({
instanceId: "smoke",
displayName: "smoke",
environment: {},
enabled: true,
config: cfg,
});

// 1. reachability
const snap = await instance.snapshot();
console.log(`[smoke] snapshot: state=${snap.state} reason=${snap.reason ?? "n/a"}`);
if (snap.state !== "available") {
console.error(`[smoke] counsel unreachable; aborting`);
await instance.dispose();
return 1;
}

// 2. send turn
let synthesis = "";
let seats: Array<{ stage: string; model: string; text: string }> = [];
let errors: string[] = [];
let stopReason: string | null = null;
let ok = false;
let sessionId: string | null = null;
let usageTotal = { input: 0, output: 0 };

const done = new Promise<void>((resolve) => {
// title is on item.started; item.completed carries the same itemId so we
// pair them up to get a labeled text for the smoke output.
const titles = new Map<string, string>();
const unsub = instance.adapter.onEvent((e) => {
switch (e.type) {
case "session.started":
sessionId = e.sessionId;
console.log(`[smoke] session.started sessionId=${e.sessionId} model=${e.model}`);
break;
case "turn.started":
console.log(`[smoke] turn.started`);
break;
case "item.started":
if (e.itemId && e.title) titles.set(e.itemId, e.title);
break;
case "item.completed":
if (e.itemType === "assistant_text" && e.itemId) {
const title = titles.get(e.itemId) ?? "seat";
const stage = title.match(/^(\S+)/)?.[1] ?? "seat";
const modelMatch = title.match(/\(([^)]+)\)$/)?.[1] ?? "unknown";
seats.push({ stage, model: modelMatch, text: e.text });
console.log(`[smoke] ${stage.padEnd(20)} [${modelMatch}] ${e.text.slice(0, 120).replace(/\n/g, "\\n")}${e.text.length > 120 ? "..." : ""}`);
// The last assistant_text is the synthesis (stage3)
synthesis = e.text;
}
break;
case "thread.token-usage.updated":
usageTotal.input += e.input;
usageTotal.output += e.output;
break;
case "turn.completed":
ok = e.ok;
stopReason = e.stopReason ?? null;
break;
case "session.exited":
console.log(`[smoke] session.exited reason=${e.reason}`);
unsub();
resolve();
break;
case "runtime.error":
errors.push(e.message);
console.error(`[smoke] runtime.error: ${e.message}`);
break;
}
});
});

await instance.adapter.sendTurn({
threadId: `smoke-${Date.now()}`,
text: prompt,
model,
});
await done;
await instance.dispose();

console.log(`[smoke] seats: ${seats.length} (stage1=${seats.filter(s => s.stage.startsWith("stage1")).length}, stage2=${seats.filter(s => s.stage.startsWith("stage2")).length}, stage3=${seats.filter(s => s.stage.startsWith("stage3")).length})`);
console.log(`[smoke] ok=${ok} stopReason=${stopReason}`);
console.log(`[smoke] usage: in=${usageTotal.input} out=${usageTotal.output}`);
if (errors.length) console.log(`[smoke] errors: ${JSON.stringify(errors)}`);
console.log(`[smoke] synthesis: ${JSON.stringify(synthesis.slice(0, 200))}`);

if (!ok || !synthesis) {
console.error(`[smoke] FAILED: ok=${ok} synthesis-empty=${!synthesis}`);
return 1;
}
if (sessionId == null) {
console.error(`[smoke] FAILED: no sessionId`);
return 1;
}
console.log(`[smoke] OK`);
return 0;
}

main().then((code) => process.exit(code)).catch((e) => {
console.error(`[smoke] crashed:`, e);
process.exit(2);
});
121 changes: 121 additions & 0 deletions scripts/smoke-hermes-os.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// End-to-end smoke test for the hermes-os driver against a live hub.
//
// Run from the OpenMausBot checkout:
// HERMES_OS_URL=http://server.tail85e19.ts.net:8001 \
// node --experimental-strip-types scripts/smoke-hermes-os.ts
//
// Or run on the homelab server itself (where :8001 is local):
// HERMES_OS_URL=http://127.0.0.1:8001 node --experimental-strip-types scripts/smoke-hermes-os.ts
//
// The script:
// 1. constructs the driver with HERMES_OS_URL
// 2. asks the hub for /v1/models to confirm reachability
// 3. sends a single turn with the default model
// 4. streams content.delta events to stdout as they arrive
// 5. prints the final turn summary
// 6. exits 0 on success, 1 on failure
//
// Designed to be safe to run from cron or a watchdog — no side effects,
// no writes to disk outside the native-log dir.

import { HermesOsDriver } from "../server/drivers/hermes-os.ts";

const baseUrl = process.env.HERMES_OS_URL ?? "http://127.0.0.1:8001";
const model = process.env.HERMES_OS_MODEL ?? "gemini";
const prompt = process.env.HERMES_OS_PROMPT ?? "Reply with exactly: hermes-os smoke ok";
const apiKey = process.env.HUB_API_TOKEN;

async function main(): Promise<number> {
console.log(`[smoke] baseUrl=${baseUrl} model=${model} auth=${apiKey ? "yes" : "no"}`);

const cfg = HermesOsDriver.decodeConfig({ baseUrl, apiKey, defaultModel: model });
const instance = await HermesOsDriver.create({
instanceId: "smoke",
displayName: "smoke",
environment: {},
enabled: true,
config: cfg,
});

// 1. reachability
const snap = await instance.snapshot();
console.log(`[smoke] snapshot: state=${snap.state} reason=${snap.reason ?? "n/a"}`);
if (snap.state !== "available") {
console.error(`[smoke] hub unreachable; aborting`);
await instance.dispose();
return 1;
}

// 2. send turn
let text = "";
let deltas = 0;
let usage: { input: number; output: number } | null = null;
let stopReason: string | null = null;
let ok = false;
let errors: string[] = [];
let sessionId: string | null = null;

const done = new Promise<void>((resolve) => {
const unsub = instance.adapter.onEvent((e) => {
switch (e.type) {
case "session.started":
sessionId = e.sessionId;
console.log(`[smoke] session.started sessionId=${e.sessionId} model=${e.model}`);
break;
case "turn.started":
console.log(`[smoke] turn.started`);
break;
case "content.delta":
deltas++;
process.stdout.write(e.delta);
break;
case "item.completed":
if (e.itemType === "assistant_text") text = e.text;
break;
case "thread.token-usage.updated":
usage = { input: e.input, output: e.output };
break;
case "turn.completed":
ok = e.ok;
stopReason = e.stopReason ?? null;
break;
case "session.exited":
console.log(`\n[smoke] session.exited reason=${e.reason}`);
unsub();
resolve();
break;
case "runtime.error":
errors.push(e.message);
break;
}
});
});

await instance.adapter.sendTurn({
threadId: `smoke-${Date.now()}`,
text: prompt,
model,
});
await done;
await instance.dispose();

console.log(`[smoke] deltas=${deltas} ok=${ok} stopReason=${stopReason} text=${JSON.stringify(text.slice(0, 200))}`);
if (usage) console.log(`[smoke] usage: in=${usage.input} out=${usage.output}`);
if (errors.length) console.log(`[smoke] errors: ${JSON.stringify(errors)}`);

if (!ok || !text) {
console.error(`[smoke] FAILED: ok=${ok} text-empty=${!text}`);
return 1;
}
if (sessionId == null) {
console.error(`[smoke] FAILED: no sessionId`);
return 1;
}
console.log(`[smoke] OK`);
return 0;
}

main().then((code) => process.exit(code)).catch((e) => {
console.error(`[smoke] crashed:`, e);
process.exit(2);
});
Loading