diff --git a/console/web/src/components/chat/ChatView.tsx b/console/web/src/components/chat/ChatView.tsx index ffa752494..6e3dce710 100644 --- a/console/web/src/components/chat/ChatView.tsx +++ b/console/web/src/components/chat/ChatView.tsx @@ -18,18 +18,20 @@ import { useConversationsCtxOptional } from '@/lib/conversations-context' import { formatStopReason } from '@/lib/format-stop-reason' import { newMessageId } from '@/lib/session-id' import { cn } from '@/lib/utils' -import type { - AssistantMessage, - Conversation, - FunctionCallMessage, - Message, - MessagePatch, - Mode, - ModelId, - ModelOption, - SystemMessage, - ThoughtMessage, - UserMessage, +import { + type AssistantMessage, + type Conversation, + DEFAULT_THINKING_LEVEL, + type FunctionCallMessage, + type Message, + type MessagePatch, + type Mode, + type ModelId, + type ModelOption, + type SystemMessage, + type ThinkingLevel, + type ThoughtMessage, + type UserMessage, } from '@/types/chat' import { Composer, type ComposerSubmitPayload } from './Composer' import { ContextUsage } from './ContextUsage' @@ -99,6 +101,9 @@ export function ChatView({ onCompactConversation, }: ChatViewProps) { const [isStreaming, setIsStreaming] = useState(false) + const [thinkingLevel, setThinkingLevel] = useState( + DEFAULT_THINKING_LEVEL, + ) const abortRef = useRef(null) const [copied, setCopied] = useState(false) const { functionEntries } = useFunctionsCatalog(backend.id) @@ -303,7 +308,7 @@ export function ChatView({ payload.text || '(attachments only)', conversation.mode, model, - { signal: controller.signal, sessionId, messageId }, + { signal: controller.signal, sessionId, messageId, thinkingLevel }, )) { switch (event.kind) { case 'thought-start': { @@ -530,6 +535,7 @@ export function ChatView({ conversation.id, conversation.mode, conversation.model, + thinkingLevel, sessionId, contextWindow, backend, @@ -680,6 +686,8 @@ export function ChatView({ functionEntries={functionEntries} permissionMode={approvalSettings.settings.mode} permissionModeLoading={!approvalSettings.loaded} + thinkingLevel={thinkingLevel} + onThinkingLevelChange={setThinkingLevel} onModeChange={(next) => onUpdateMode(conversation.id, next)} onModelChange={(next) => onUpdateModel(conversation.id, next)} onPermissionModeChange={(next) => diff --git a/console/web/src/components/chat/Composer.stories.tsx b/console/web/src/components/chat/Composer.stories.tsx index 2248c3d1b..883ec9012 100644 --- a/console/web/src/components/chat/Composer.stories.tsx +++ b/console/web/src/components/chat/Composer.stories.tsx @@ -69,6 +69,8 @@ function ComposerHarness({ modelOptions={STORY_MODEL_OPTIONS} functionEntries={STATIC_FUNCTIONS} permissionMode="manual" + thinkingLevel="off" + onThinkingLevelChange={fn()} onModeChange={setMode} onModelChange={setModel} onPermissionModeChange={fn()} diff --git a/console/web/src/components/chat/Composer.tsx b/console/web/src/components/chat/Composer.tsx index 3f4f0d451..5df1332a6 100644 --- a/console/web/src/components/chat/Composer.tsx +++ b/console/web/src/components/chat/Composer.tsx @@ -4,7 +4,15 @@ import { PermissionModePicker } from '@/components/permissions/PermissionModePic import { Button } from '@/components/ui/Button' import type { PermissionMode } from '@/lib/backend/approval-settings' import type { FunctionEntry } from '@/lib/functions' -import type { Attachment, Mode, ModelId, ModelOption } from '@/types/chat' +import { Select } from '@/components/ui/Select' +import { + type Attachment, + type Mode, + type ModelId, + type ModelOption, + THINKING_LEVELS, + type ThinkingLevel, +} from '@/types/chat' import { AttachmentButton } from './AttachmentButton' import { AttachmentChip } from './AttachmentChip' import { LexicalShell } from './LexicalShell' @@ -28,8 +36,10 @@ interface ComposerProps { */ permissionMode: PermissionMode permissionModeLoading?: boolean + thinkingLevel: ThinkingLevel onModeChange: (next: Mode) => void onModelChange: (next: ModelId) => void + onThinkingLevelChange: (next: ThinkingLevel) => void onPermissionModeChange: (next: PermissionMode) => void onSubmit: (payload: ComposerSubmitPayload) => void onStop?: () => void @@ -52,8 +62,10 @@ export function Composer({ catalogLoading, permissionMode, permissionModeLoading, + thinkingLevel, onModeChange, onModelChange, + onThinkingLevelChange, onPermissionModeChange, onSubmit, onStop, @@ -133,6 +145,16 @@ export function Composer({ disabled={inputDisabled || !!permissionModeLoading} />
+ + value={thinkingLevel} + options={THINKING_LEVELS.map((l) => ({ + value: l, + label: l === 'off' ? 'thinking off' : `thinking ${l}`, + }))} + onChange={onThinkingLevelChange} + disabled={inputDisabled} + aria-label="thinking level" + /> { - window.location.hash = '#/configuration/workers/harness' + window.location.hash = '#/configuration/workers/llm-router' }, } diff --git a/console/web/src/components/chat/ModelPicker.tsx b/console/web/src/components/chat/ModelPicker.tsx index c0e8a41bd..f90a93095 100644 --- a/console/web/src/components/chat/ModelPicker.tsx +++ b/console/web/src/components/chat/ModelPicker.tsx @@ -9,10 +9,10 @@ import { type ModelOption, } from '@/types/chat' -// Deep link to the harness configuration entry in the workers/config editor, +// Deep link to the llm-router configuration entry in the workers/config editor, // where api keys + per-provider settings are now edited (the bespoke // per-provider dialog was retired in favour of the schema-driven form). -const HARNESS_CONFIG_HASH = '#/configuration/workers/harness' +const HARNESS_CONFIG_HASH = '#/configuration/workers/llm-router' interface ModelPickerProps { value: ModelId | null @@ -47,7 +47,7 @@ export function ModelPicker({ // Optional: present in the app, absent in isolated Storybook renders. const ctx = useConversationsCtxOptional() - // Providers present as harness workers (from harness::provider::list). + // Providers present as workers (from router::provider::list). // Absent in Storybook or before the list resolves, in which case no empty // provider groups or gears appear until the dynamic list arrives. const presentIds = ctx?.presentProviders.map((p) => p.id) ?? [] diff --git a/console/web/src/components/chat/ThoughtMessage.tsx b/console/web/src/components/chat/ThoughtMessage.tsx index c73cb5546..ed95a8295 100644 --- a/console/web/src/components/chat/ThoughtMessage.tsx +++ b/console/web/src/components/chat/ThoughtMessage.tsx @@ -15,8 +15,13 @@ function thoughtLabel(durationMs: number): string { export function ThoughtMessage({ message, defaultOpen }: ThoughtMessageProps) { const streaming = !!message.streaming + // Auto-open while the thought streams so reasoning is visible in real + // time; the flip back when streaming ends collapses it to its summary. return ( -
+
void @@ -13,7 +13,7 @@ export function onHarnessConfigSaved(listener: Listener): () => void { } export function notifyHarnessConfigSaved(configId: string): void { - if (configId !== HARNESS_CONFIG_ID) return + if (configId !== LLM_ROUTER_CONFIG_ID) return for (const listener of listeners) { listener() } diff --git a/console/web/src/lib/models-catalog.ts b/console/web/src/lib/models-catalog.ts index 4da0cf837..d22f7c112 100644 --- a/console/web/src/lib/models-catalog.ts +++ b/console/web/src/lib/models-catalog.ts @@ -2,7 +2,7 @@ import { makeCatalogModelKey } from '@/lib/catalog-model-key' import { getIiiClient } from '@/lib/iii-client' import type { ModelOption } from '@/types/chat' -/** Wire shape returned by `models::list` over the iii bus. */ +/** Wire shape returned by `router::models::list` over the iii bus. */ export interface CatalogModelRow { id: string provider: string @@ -12,7 +12,7 @@ export interface CatalogModelRow { export async function fetchModelsCatalog(): Promise { const client = await getIiiClient() - const res = await client.call<{ models?: unknown }>('models::list', {}) + const res = await client.call<{ models?: unknown }>('router::models::list', {}) const rows = res?.models if (!Array.isArray(rows)) return [] const out: CatalogModelRow[] = [] @@ -50,7 +50,7 @@ export function catalogRowsToModelOptions( * Ask each provider to re-pull its upstream model list into the catalog via * `provider::::refresh_models`. Best-effort and parallel — a provider * that's offline or has no credential simply registers nothing. Callers - * re-read `models::list` afterwards to pick up the refreshed catalog. + * re-read `router::models::list` afterwards to pick up the refreshed catalog. */ export async function refreshProviderModels( providers: readonly string[], @@ -96,7 +96,7 @@ export async function subscribeModelChanges( } } -/** A provider declared to the harness, from `harness::provider::list`. */ +/** A provider declared to the harness, from `router::provider::list`. */ export interface ProviderListEntry { id: string display_name: string @@ -111,7 +111,7 @@ export interface ProviderListEntry { export async function fetchProviderList(): Promise { const client = await getIiiClient() const res = await client.call<{ providers?: unknown }>( - 'harness::provider::list', + 'router::provider::list', {}, ) const rows = res?.providers diff --git a/console/web/src/lib/providers.ts b/console/web/src/lib/providers.ts index aa9b4a212..1e88fdfe1 100644 --- a/console/web/src/lib/providers.ts +++ b/console/web/src/lib/providers.ts @@ -1,7 +1,7 @@ /** * Client-side validation helpers + error normalization shared by provider * surfaces. Credentials and per-provider settings are now edited through the - * schema-driven `configuration` form (harness entry), so the former + * schema-driven `configuration` form (llm-router entry), so the former * `auth::*` / `provider_config::*` bus wrappers were removed — the * `configuration::*` calls in `WorkersTab/api.ts` cover that path. */ diff --git a/console/web/src/pages/Configuration/tabs/ConsoleSettingsTab.tsx b/console/web/src/pages/Configuration/tabs/ConsoleSettingsTab.tsx index 39ac73050..68417240f 100644 --- a/console/web/src/pages/Configuration/tabs/ConsoleSettingsTab.tsx +++ b/console/web/src/pages/Configuration/tabs/ConsoleSettingsTab.tsx @@ -19,9 +19,9 @@ import { saveDefaultAllowlist, } from '@/lib/storage' -// Provider credentials + settings now live in the harness `configuration` +// Provider credentials + settings now live in the llm-router `configuration` // entry, edited via the schema-driven form on the workers tab. -const HARNESS_CONFIG_HASH = '#/configuration/workers/harness' +const HARNESS_CONFIG_HASH = '#/configuration/workers/llm-router' interface ConsoleSettingsTabProps { theme: Theme diff --git a/console/web/src/types/chat.ts b/console/web/src/types/chat.ts index 601155a90..acdfc6569 100644 --- a/console/web/src/types/chat.ts +++ b/console/web/src/types/chat.ts @@ -19,6 +19,20 @@ export const MODES: { id: Mode; label: string }[] = [ export const DEFAULT_MODE: Mode = 'agent' +/** Reasoning effort sent to run::start as `thinking_level`; 'off' is omitted. */ +export type ThinkingLevel = 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' + +export const THINKING_LEVELS: ThinkingLevel[] = [ + 'off', + 'minimal', + 'low', + 'medium', + 'high', + 'xhigh', +] + +export const DEFAULT_THINKING_LEVEL: ThinkingLevel = 'off' + export type Role = 'user' | 'assistant' | 'thought' | 'function-call' export interface Attachment { diff --git a/harness/package.json b/harness/package.json index 4a58d585a..47e0fbba6 100644 --- a/harness/package.json +++ b/harness/package.json @@ -2,7 +2,7 @@ "name": "harness", "version": "0.5.7", "private": true, - "description": "Node port of the iii harness stack: harness, approval-gate, turn-orchestrator, llm-budget, providers, and side-cars. Conversations persist in the external session-manager worker.", + "description": "Node port of the iii harness stack: harness, approval-gate, turn-orchestrator, llm-budget, and side-cars. Conversations persist in the external session-manager worker; LLM providers are standalone llm-router plugin workers.", "license": "Apache-2.0", "type": "module", "engines": { @@ -26,12 +26,6 @@ "dev:turn-orchestrator": "tsx src/turn-orchestrator/main.ts", "dev:llm-budget": "tsx src/llm-budget/main.ts", "dev:hook-fanout": "tsx src/hook-fanout/main.ts", - "dev:models-catalog": "tsx src/models-catalog/main.ts", - "dev:provider-anthropic": "tsx src/provider-anthropic/main.ts", - "dev:provider-openai": "tsx src/provider-openai/main.ts", - "dev:provider-kimi": "tsx src/provider-kimi/main.ts", - "dev:provider-lmstudio": "tsx src/provider-lmstudio/main.ts", - "dev:provider-llamacpp": "tsx src/provider-llamacpp/main.ts", "dev:context-compaction": "tsx src/context-compaction/main.ts", "dev:web": "tsx src/web/main.ts" }, @@ -42,12 +36,6 @@ "iii-turn-orchestrator": "./dist/turn-orchestrator/main.js", "iii-llm-budget": "./dist/llm-budget/main.js", "iii-hook-fanout": "./dist/hook-fanout/main.js", - "iii-models-catalog": "./dist/models-catalog/main.js", - "iii-provider-anthropic": "./dist/provider-anthropic/main.js", - "iii-provider-openai": "./dist/provider-openai/main.js", - "iii-provider-kimi": "./dist/provider-kimi/main.js", - "iii-provider-lmstudio": "./dist/provider-lmstudio/main.js", - "iii-provider-llamacpp": "./dist/provider-llamacpp/main.js", "iii-context-compaction": "./dist/context-compaction/main.js", "iii-web": "./dist/web/main.js" }, diff --git a/harness/src/context-compaction/model-resolver.ts b/harness/src/context-compaction/model-resolver.ts index 22ebb2781..6b4f5ea69 100644 --- a/harness/src/context-compaction/model-resolver.ts +++ b/harness/src/context-compaction/model-resolver.ts @@ -30,21 +30,26 @@ export async function fetchModelLimit( modelID: string, ): Promise { try { + // router::models::get: payload key is `id`, result is wrapped as + // `{ model }`, null on a catalog miss (the cold-window signal). const entry = await iii.trigger< unknown, { - id?: string; - provider?: string; - context_window?: number; - max_output_tokens?: number; + model?: { + id?: string; + provider?: string; + context_window?: number; + max_output_tokens?: number; + } | null; } | null >({ - function_id: 'models::get', - payload: { provider: providerID, model_id: modelID }, + function_id: 'router::models::get', + payload: { provider: providerID, id: modelID }, timeoutMs: 5_000, }); - if (!entry) { + const model = entry?.model ?? null; + if (!model) { logger.debug('model-resolver: model not found in catalog', { providerID, modelID }); return null; } @@ -52,10 +57,10 @@ export async function fetchModelLimit( return { providerID, modelID, - modelLimit: limitFromModel(entry), + modelLimit: limitFromModel(model), }; } catch (err) { - logger.debug('model-resolver: models::get failed', { + logger.debug('model-resolver: router::models::get failed', { providerID, modelID, err: String(err), diff --git a/harness/src/context-compaction/stream-collect.ts b/harness/src/context-compaction/stream-collect.ts deleted file mode 100644 index be77213f5..000000000 --- a/harness/src/context-compaction/stream-collect.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { ISdk, StreamChannelRef } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import type { AssistantMessage } from '../types/agent-message.js'; -import type { ProviderStreamInput } from '../types/provider.js'; -import type { AssistantMessageEvent } from '../types/stream-event.js'; - -const SUMMARIZER_TIMEOUT_MS = 120_000; - -export type StreamCollectInput = Omit; - -export async function streamAndCollect( - iii: ISdk, - input: StreamCollectInput, - providerFunctionId: string, -): Promise { - const channel = await iii.createChannel(); - const events: AssistantMessageEvent[] = []; - let resolveNext: (() => void) | null = null; - let terminal: AssistantMessageEvent | null = null; - - channel.reader.onMessage((raw: string) => { - try { - const ev = JSON.parse(raw) as AssistantMessageEvent; - events.push(ev); - if (ev.type === 'done' || ev.type === 'error') terminal = ev; - if (resolveNext) { - const fn = resolveNext; - resolveNext = null; - fn(); - } - } catch (err) { - logger.warn('streamAndCollect: decode failed', { err: String(err) }); - } - }); - // iii-sdk@0.12.0: onMessage doesn't open the read-side; resume() does. - channel.reader.stream.resume(); - - await iii.trigger({ - function_id: providerFunctionId, - payload: { - ...input, - writer_ref: channel.writerRef satisfies StreamChannelRef, - }, - timeoutMs: SUMMARIZER_TIMEOUT_MS, - }); - - // trigger() resolved, but the channel may not have delivered the - // terminal event yet. Poll for up to GRACE_MS before giving up so a - // slow IPC hop doesn't masquerade as "stream returned without a - // terminal event". - const GRACE_MS = 1_000; - const deadline = Date.now() + GRACE_MS; - while (!terminal && Date.now() < deadline) { - await new Promise((r) => { - resolveNext = r; - setTimeout(r, 25); - }); - } - - if (!terminal) { - throw new Error('summariser stream returned without a terminal event'); - } - if ((terminal as AssistantMessageEvent).type === 'error') { - // Surface provider errors as a thrown exception so summarizeAndAppend's - // catch treats them as compaction failures. Without this, the error - // AssistantMessage's text content got silently written as the summary. - const errMsg = (terminal as { type: 'error'; error: AssistantMessage }).error; - const detail = - typeof errMsg.error_message === 'string' && errMsg.error_message.length > 0 - ? errMsg.error_message - : extractTextFromMessage(errMsg); - throw new Error(`summariser stream error: ${detail || 'unknown provider error'}`); - } - return (terminal as { type: 'done'; message: AssistantMessage }).message; -} - -function extractTextFromMessage(msg: AssistantMessage): string { - for (const block of msg.content ?? []) { - if ((block as { type?: string }).type === 'text') { - return (block as { type: 'text'; text: string }).text; - } - } - return ''; -} diff --git a/harness/src/context-compaction/summarize.ts b/harness/src/context-compaction/summarize.ts index 3ab8c3df7..9d815aa14 100644 --- a/harness/src/context-compaction/summarize.ts +++ b/harness/src/context-compaction/summarize.ts @@ -8,7 +8,6 @@ import { readActivePath, sessionAppendCustom, } from '../runtime/session.js'; -import { decide, targetFunctionId } from '../turn-orchestrator/provider-router.js'; import type { AgentMessage, AssistantMessage } from '../types/agent-message.js'; import { compactionConfig } from './config.js'; import { stampLastCompaction } from './lease.js'; @@ -19,10 +18,12 @@ import { completedCompactions, selectWithEntryIds, } from './selection.js'; -import { streamAndCollect } from './stream-collect.js'; import { stripMedia } from './strip-media.js'; import { buildPrompt } from './template.js'; +/** Outer trigger budget; must exceed the router's 300s stream budget. */ +const SUMMARIZER_TIMEOUT_MS = 320_000; + export type SummarizeMode = 'async' | 'sync'; export type SummarizeOptions = { @@ -157,19 +158,14 @@ export async function summarizeAndAppend( const systemPrompt = buildPrompt({ previousSummary, context: [] }); const userPrompt = renderUserPrompt(stripped); - // Always use the session's own provider/model; route through the - // canonical provider-router so adding a provider covers /compact too. - const summariserId = targetFunctionId( - decide({ provider: model.providerID, model: model.modelID }), - ); - const modelId = model.modelID; - // One 250ms retry for transient failures (429/5xx/network). Permanent // failures (auth, malformed request) skip the retry to release the lease - // sooner. - const streamInput = { + // sooner. The session's own provider/model are pinned explicitly so + // /compact runs on exactly the provider the session streams on. + const completeInput = { + model: model.modelID, + provider: model.providerID, system_prompt: systemPrompt, - model: modelId, messages: [ { role: 'user' as const, @@ -181,7 +177,7 @@ export async function summarizeAndAppend( }; let final: AssistantMessage; try { - final = await streamAndCollect(iii, streamInput, summariserId); + final = await routerComplete(iii, completeInput); } catch (firstErr) { const firstReason = firstErr instanceof Error ? firstErr.message : String(firstErr); if (!isRetryableStreamError(firstErr)) { @@ -193,7 +189,7 @@ export async function summarizeAndAppend( setTimeout(resolve, 250); }); try { - final = await streamAndCollect(iii, streamInput, summariserId); + final = await routerComplete(iii, completeInput); } catch (secondErr) { const reason = secondErr instanceof Error ? secondErr.message : String(secondErr); logger.warn('summariser stream failed after retry', { @@ -235,3 +231,41 @@ export async function summarizeAndAppend( tail_messages, }; } + +/** + * Run the summariser turn through `router::complete`. Pre-stream failures + * (unknown provider, not configured) throw from the trigger; a mid-stream + * failure comes back as an error-shaped AssistantMessage, surfaced here as a + * throw so the caller's retry logic treats both uniformly. + */ +async function routerComplete( + iii: ISdk, + input: Record, +): Promise { + const resp = await iii.trigger({ + function_id: 'router::complete', + payload: input, + timeoutMs: SUMMARIZER_TIMEOUT_MS, + }); + const message = resp?.message; + if (!message) { + throw new Error('router::complete returned no message'); + } + if (message.stop_reason === 'error') { + const detail = + typeof message.error_message === 'string' && message.error_message.length > 0 + ? message.error_message + : extractTextFromMessage(message); + throw new Error(`summariser stream error: ${detail || 'unknown provider error'}`); + } + return message; +} + +function extractTextFromMessage(msg: AssistantMessage): string { + for (const block of msg.content ?? []) { + if ((block as { type?: string }).type === 'text') { + return (block as { type: 'text'; text: string }).text; + } + } + return ''; +} diff --git a/harness/src/harness/fanout/models-changed.ts b/harness/src/harness/fanout/models-changed.ts index bf025ba87..af5a7db89 100644 --- a/harness/src/harness/fanout/models-changed.ts +++ b/harness/src/harness/fanout/models-changed.ts @@ -1,4 +1,3 @@ -import { MODELS_SCOPE } from '../../models-catalog/types.js'; import type { ISdk, Trigger } from '../../runtime/iii.js'; import { logger } from '../../runtime/otel.js'; import type { FanoutState } from '../ui-subscribe.js'; @@ -13,8 +12,8 @@ const DEBOUNCE_MS = 250; /** * Push `ui::models::changed::` to every subscribed browser. - * Called explicitly after a config-driven refresh wave and by the debounced - * state-trigger handler after catalog writes. + * Called by the debounced `router::models::changed` handler after catalog + * writes. */ export function emitModelsCatalogChanged(iii: ISdk, state: FanoutState): void { for (const browser_id of state.modelSubscribers()) { @@ -32,9 +31,10 @@ export function emitModelsCatalogChanged(iii: ISdk, state: FanoutState): void { } /** - * The model catalog lives in iii state (scope `models`, one `Model[]` per - * provider key). A `state` trigger notifies browsers after writes; trailing - * debounce collapses overlapping provider reconciles into one push. + * The model catalog lives in the llm-router worker, which publishes + * `router::models::changed` on every reconcile. A subscribe trigger notifies + * browsers after writes; trailing debounce collapses overlapping provider + * reconciles into one push. */ export function spawnModelsChanged(iii: ISdk, state: FanoutState): () => void { let timer: ReturnType | null = null; @@ -57,19 +57,19 @@ export function spawnModelsChanged(iii: ISdk, state: FanoutState): () => void { }, { description: - 'Internal: coalesces models-scope state changes into ui::models::changed:: pushes.', + 'Internal: coalesces router::models::changed events into ui::models::changed:: pushes.', }, ); let trigger: Trigger | null = null; try { trigger = iii.registerTrigger({ - type: 'state', + type: 'subscribe', function_id: MODELS_CHANGED_HANDLER_FN_ID, - config: { scope: MODELS_SCOPE }, + config: { topic: 'router::models::changed' }, }); } catch (err) { - logger.warn('models state trigger registration failed', { err: String(err) }); + logger.warn('router::models::changed trigger registration failed', { err: String(err) }); } return () => { diff --git a/harness/src/harness/iii.worker.yaml b/harness/src/harness/iii.worker.yaml index f858de8d8..a74888252 100644 --- a/harness/src/harness/iii.worker.yaml +++ b/harness/src/harness/iii.worker.yaml @@ -22,10 +22,8 @@ dependencies: iii-sandbox: "^0.11.0" iii-directory: "^0.5.1" turn-orchestrator: "^0.2.0" - models-catalog: "^0.2.0" + llm-router: "^0.1.0" shell: "^0.3.0" - provider-anthropic: "^0.2.0" - provider-openai: "^0.2.0" approval-gate: "^0.2.0" session: "^0.2.0" hook-fanout: "^0.2.0" diff --git a/harness/src/harness/migrate-llm-router-config.ts b/harness/src/harness/migrate-llm-router-config.ts new file mode 100644 index 000000000..c2ebe7275 --- /dev/null +++ b/harness/src/harness/migrate-llm-router-config.ts @@ -0,0 +1,112 @@ +/** + * One-time boot migration: copy the `harness` entry's `providers` block into + * the `llm-router` configuration entry (now the single home for provider + * credentials + settings), and seed routing parity with the legacy local + * `decide()` — anthropic as the default provider plus the gpt-/o- and + * kimi-/moonshot-v1- prefix heuristics. + * + * Idempotent: a marker in iii-state (scope `harness-migrations`) skips the + * work on later boots, and an `llm-router` entry that already has providers + * is treated as migrated (the operator got there first). The copy reads the + * `harness` entry raw so `${VAR:default}` templates survive verbatim. The set + * is retried briefly because the router composes the entry schema only after + * the first provider registers. + * + * The stale `providers` block in the `harness` entry is left in place; the + * console edits the `llm-router` entry from now on. + */ + +import { configurationGet, configurationSet, type JsonValue } from '../runtime/configuration.js'; +import type { ISdk } from '../runtime/iii.js'; +import { logger } from '../runtime/otel.js'; +import { stateGet, stateSet } from '../runtime/state.js'; + +export const LLM_ROUTER_CONFIG_ID = 'llm-router'; + +const MIGRATION_SCOPE = 'harness-migrations'; +const MIGRATION_KEY = 'llm-router-providers'; + +const SET_ATTEMPTS = 10; +const SET_RETRY_MS = 3_000; + +/** + * Routing parity with the legacy `decide()` (turn-orchestrator + * provider-router): unmatched models fell back to anthropic, gpt-/o- + * routed to openai, kimi-/moonshot-v1- to kimi. Rust regex: inline `(?i)`, + * no `/i` flag. + */ +export const ROUTING_PARITY_SEED = { + default_provider: 'anthropic', + routing_heuristics: [ + { pattern: '(?i)^(gpt-|o[0-9]-)', provider: 'openai' }, + { pattern: '(?i)^(kimi-|moonshot-v1-)', provider: 'kimi' }, + ], +} as const; + +function asObject(v: JsonValue | null): Record { + return v && typeof v === 'object' && !Array.isArray(v) ? { ...v } : {}; +} + +function providersOf(v: JsonValue | null): Record { + const providers = asObject(v).providers; + return asObject(providers ?? null); +} + +/** Compose the migrated `llm-router` value without clobbering operator edits. */ +export function composeMigratedValue( + routerValue: JsonValue | null, + harnessProviders: Record, +): Record { + const out = asObject(routerValue); + out.providers = { ...harnessProviders, ...providersOf(routerValue) }; + if (typeof out.default_provider !== 'string') { + out.default_provider = ROUTING_PARITY_SEED.default_provider; + } + if (!Array.isArray(out.routing_heuristics)) { + out.routing_heuristics = ROUTING_PARITY_SEED.routing_heuristics.map((h) => ({ ...h })); + } + return out; +} + +export async function migrateLlmRouterConfig(iii: ISdk): Promise { + const marker = await stateGet<{ migrated_at?: string }>(iii, MIGRATION_SCOPE, MIGRATION_KEY); + if (marker) return; + + const routerValue = await configurationGet(iii, LLM_ROUTER_CONFIG_ID, { raw: true }); + if (Object.keys(providersOf(routerValue)).length > 0) { + // Operator already populated the entry — record that and never touch it again. + await stateSet(iii, MIGRATION_SCOPE, MIGRATION_KEY, { + migrated_at: new Date().toISOString(), + note: 'llm-router entry already populated', + }); + return; + } + + // Raw read preserves `${VAR:default}` credential templates verbatim. + const harnessValue = await configurationGet(iii, 'harness', { raw: true }); + const migrated = composeMigratedValue(routerValue, providersOf(harnessValue)); + + for (let attempt = 1; attempt <= SET_ATTEMPTS; attempt++) { + try { + await configurationSet(iii, LLM_ROUTER_CONFIG_ID, migrated); + await stateSet(iii, MIGRATION_SCOPE, MIGRATION_KEY, { + migrated_at: new Date().toISOString(), + providers: Object.keys(providersOf(migrated)), + }); + logger.info('migrated provider config to the llm-router entry', { + providers: Object.keys(providersOf(migrated)), + }); + return; + } catch (err) { + // The router registers the entry (and composes its schema) only after + // the first provider registration lands — early boots race that. + logger.warn('llm-router config migration set failed; retrying', { + attempt, + err: String(err), + }); + await new Promise((resolve) => setTimeout(resolve, SET_RETRY_MS)); + } + } + // No marker written: the next boot retries the whole migration. + logger.error('llm-router config migration did not land; will retry next boot', {}); +} diff --git a/harness/src/harness/permissions-config.ts b/harness/src/harness/permissions-config.ts new file mode 100644 index 000000000..ed392bffc --- /dev/null +++ b/harness/src/harness/permissions-config.ts @@ -0,0 +1,71 @@ +/** + * Registers the `harness` configuration entry — permissions only. Provider + * credentials/settings moved to the `llm-router` entry, whose schema the + * router composes from provider declarations; this entry keeps the agent + * permissions block (and nothing else) editable in the console. + * + * Root `additionalProperties` stays true so a stale `providers` block left + * behind by the pre-router layout never fails value validation; the migration + * (`migrate-llm-router-config.ts`) copies it, it just stops being editable. + */ + +import { + configurationGet, + configurationRegister, + type JsonValue, +} from '../runtime/configuration.js'; +import { + DEFAULT_PERMISSION_MODE, + HARNESS_CONFIG_ID, + PERMISSION_MODES, +} from '../runtime/harness-config.js'; +import type { ISdk } from '../runtime/iii.js'; +import { logger } from '../runtime/otel.js'; + +const ENTRY_DESCRIPTION = 'Agent permissions, managed by the harness.'; + +export async function registerHarnessConfigEntry(iii: ISdk): Promise { + const schema = { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + title: HARNESS_CONFIG_ID, + properties: { + permissions: { + type: 'object', + title: 'permissions', + properties: { + default_mode: { + type: 'string', + title: 'default mode', + description: 'Default approval mode applied to new agent sessions.', + enum: [...PERMISSION_MODES], + default: DEFAULT_PERMISSION_MODE, + }, + }, + required: ['default_mode'], + additionalProperties: false, + }, + }, + required: ['permissions'], + additionalProperties: true, + }; + + try { + // Read the stored template form so re-registration preserves any + // operator-set value; only seed on the very first registration. + const existing = await configurationGet(iii, HARNESS_CONFIG_ID, { raw: true }); + await configurationRegister(iii, { + id: HARNESS_CONFIG_ID, + name: HARNESS_CONFIG_ID, + description: ENTRY_DESCRIPTION, + schema, + ...(existing === null && { + initial_value: { + permissions: { default_mode: DEFAULT_PERMISSION_MODE }, + } as unknown as JsonValue, + }), + }); + } catch (err) { + logger.warn('harness: configuration::register failed', { err: String(err) }); + } +} diff --git a/harness/src/harness/providers/refresh-on-config.ts b/harness/src/harness/providers/refresh-on-config.ts deleted file mode 100644 index ab281ac52..000000000 --- a/harness/src/harness/providers/refresh-on-config.ts +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Bridge harness-config edits to model re-discovery. - * - * Editing the `harness` configuration entry (adding/removing a provider api - * key in the console Workers tab, or via `configuration::set` from a script) - * should make the model picker reflect the change without a manual refresh. - * Only providers whose discovery-relevant settings changed are refreshed. - * A single `ui::models::changed` push runs after the refresh wave completes. - */ - -import { emitModelsCatalogChanged } from '../fanout/models-changed.js'; -import { - HARNESS_CONFIG_ID, - normalizeHarnessConfig, - parseConfigurationChangeEvent, - providersAffectedByConfigChange, -} from '../../runtime/harness-config.js'; -import type { ISdk } from '../../runtime/iii.js'; -import { logger } from '../../runtime/otel.js'; -import type { FanoutState } from '../ui-subscribe.js'; -import type { ProviderRegistry } from './registry.js'; - -const HANDLER_FN_ID = 'harness::providers::refresh_on_config'; - -/** Coalesce window for rapid successive config writes. */ -const DEBOUNCE_MS = 500; - -/** Per-provider refresh budget — discovery hits an upstream `/v1/models`. */ -const REFRESH_TIMEOUT_MS = 30_000; - -export function registerProviderRefreshOnConfig( - iii: ISdk, - registry: ProviderRegistry, - fanoutState: FanoutState, -): void { - let timer: ReturnType | null = null; - let pendingEvent: ReturnType | null = null; - - const refreshProviders = async (providerIds: readonly string[]): Promise => { - const listing = new Set( - registry - .list() - .filter((p) => p.supports_model_listing) - .map((p) => p.id), - ); - const targets = providerIds.filter((id) => listing.has(id)); - if (targets.length === 0) return; - - await Promise.allSettled( - targets.map((id) => - iii - .trigger({ - function_id: `provider::${id}::refresh_models`, - payload: {}, - timeoutMs: REFRESH_TIMEOUT_MS, - }) - .catch((err) => - logger.debug('provider refresh-on-config failed', { - provider: id, - err: String(err), - }), - ), - ), - ); - emitModelsCatalogChanged(iii, fanoutState); - }; - - const runDebounced = async (): Promise => { - timer = null; - const event = pendingEvent; - pendingEvent = null; - if (!event) return; - - const oldCfg = normalizeHarnessConfig(event.old_value); - const newCfg = normalizeHarnessConfig(event.new_value); - - let targets: string[]; - if (event.old_value === null && event.new_value === null) { - targets = registry - .list() - .filter((p) => p.supports_model_listing) - .map((p) => p.id); - } else { - targets = providersAffectedByConfigChange(oldCfg, newCfg); - } - - await refreshProviders(targets); - }; - - try { - iii.registerFunction(HANDLER_FN_ID, async (payload: unknown) => { - pendingEvent = parseConfigurationChangeEvent(payload); - if (timer !== null) clearTimeout(timer); - timer = setTimeout(() => void runDebounced(), DEBOUNCE_MS); - return null; - }); - iii.registerTrigger({ - type: 'configuration', - function_id: HANDLER_FN_ID, - config: { configuration_id: HARNESS_CONFIG_ID }, - }); - } catch (err) { - logger.warn('harness: could not bind provider refresh-on-config trigger', { - err: String(err), - }); - } -} diff --git a/harness/src/harness/providers/register.ts b/harness/src/harness/providers/register.ts deleted file mode 100644 index de460c4a0..000000000 --- a/harness/src/harness/providers/register.ts +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Bus surface for the harness provider registry: - * - * - `harness::provider::register` — a provider self-declares its config - * schema + defaults; the registry recomposes and re-registers the - * `harness` configuration entry. - * - `harness::provider::resolve` — a provider fetches its credential + - * settings at request time (secret stays server-side; agents are denied - * this function in `iii-permissions.yaml`). - * - `harness::provider::list` — enumerate declared providers (id, - * display_name, supports_model_listing) for the console. - */ - -import { z } from 'zod'; -import type { ISdk } from '../../runtime/iii.js'; -import { logger } from '../../runtime/otel.js'; -import { ProviderRegistry } from './registry.js'; - -const DeclarationSchema = z.object({ - id: z.string().min(1), - display_name: z.string().optional(), - credential_env_var: z.string().optional(), - config_schema: z.record(z.unknown()).optional(), - defaults: z.record(z.unknown()).optional(), - supports_model_listing: z.boolean().optional(), -}); - -const ResolveSchema = z.object({ provider: z.string().min(1) }); - -/** - * Construct the registry, seed the base `harness` entry, and register the - * three bus functions. Returns the registry so callers can reuse it. - */ -export async function registerProviderRegistry(iii: ISdk): Promise { - const registry = new ProviderRegistry(iii); - await registry.init(); - - iii.registerFunction( - 'harness::provider::register', - async (payload: unknown) => { - const decl = DeclarationSchema.parse(payload); - await registry.declare(decl); - return { ok: true }; - }, - { - description: - 'Self-declare an LLM provider (id, config schema, defaults) into the dynamic harness configuration schema.', - }, - ); - - iii.registerFunction( - 'harness::provider::resolve', - async (payload: unknown) => { - const { provider } = ResolveSchema.parse(payload); - return registry.resolve(provider); - }, - { - description: - 'Resolve a provider credential + settings (api_url, max_tokens) from the harness configuration. Server-side only.', - }, - ); - - iii.registerFunction('harness::provider::list', async () => ({ providers: registry.list() }), { - description: 'List providers declared to the harness.', - }); - - logger.info('provider-registry: ready', {}); - return registry; -} diff --git a/harness/src/harness/providers/registry.ts b/harness/src/harness/providers/registry.ts deleted file mode 100644 index f2a9d86e8..000000000 --- a/harness/src/harness/providers/registry.ts +++ /dev/null @@ -1,245 +0,0 @@ -/** - * In-memory provider registry that owns the `harness` configuration entry. - * - * Each provider worker self-declares (`harness::provider::register`) its - * config schema + defaults at startup. The registry composes the aggregate - * JSON Schema and (re-)registers the `harness` entry in the `configuration` - * worker, so the editable shape grows/shrinks with the set of live providers. - * - * Providers fetch their secret + settings at request time via - * `harness::provider::resolve`, which reads the stored value and falls back - * to the provider's env var when no `api_key` is configured. - * - * Declarations are serialized through an in-process queue so concurrent - * startup declarations from sibling provider workers can't clobber each - * other's schema/value writes — the harness composite is the single owner. - */ - -import { - configurationGet, - configurationRegister, - type JsonSchema, - type JsonValue, -} from '../../runtime/configuration.js'; -import { - baseHarnessConfigValue, - DEFAULT_PERMISSION_MODE, - HARNESS_CONFIG_ID, - type HarnessConfigValue, - type HarnessProviderConfig, - normalizeHarnessConfig, - PERMISSION_MODES, -} from '../../runtime/harness-config.js'; -import type { ISdk } from '../../runtime/iii.js'; -import { logger } from '../../runtime/otel.js'; -import type { Credential } from '../../runtime/provider-resolve.js'; - -const ENTRY_NAME = 'harness'; -const ENTRY_DESCRIPTION = - 'LLM provider credentials/settings and agent permissions, managed by the harness.'; - -export type ProviderDefaults = { - api_url?: string; - max_tokens?: number; -} & Record; - -/** Payload a provider sends to `harness::provider::register`. */ -export type ProviderDeclaration = { - id: string; - display_name?: string; - /** Env var consulted as a credential fallback when no `api_key` is configured. */ - credential_env_var?: string; - /** JSON Schema for this provider's config object (api_key, api_url, ...). */ - config_schema?: JsonSchema; - /** Default settings, used to seed the value and as a resolve() fallback. */ - defaults?: ProviderDefaults; - /** True when the provider exposes `provider::::refresh_models`. */ - supports_model_listing?: boolean; -}; - -/** Result of `harness::provider::resolve`. */ -export type ProviderResolveResult = { - configured: boolean; - source: 'stored' | 'environment' | null; - credential: Credential | null; - api_url: string | null; - max_tokens: number | null; -}; - -export type ProviderListEntry = { - id: string; - display_name: string; - supports_model_listing: boolean; -}; - -/** Generic per-provider schema used when a provider declares without one. */ -function defaultProviderSchema(decl: ProviderDeclaration): JsonSchema { - return { - type: 'object', - title: decl.display_name ?? decl.id, - properties: { - api_key: { type: 'string', title: 'api key', format: 'password', writeOnly: true }, - api_url: { - type: 'string', - title: 'api url', - ...(decl.defaults?.api_url !== undefined && { default: decl.defaults.api_url }), - }, - max_tokens: { - type: 'integer', - title: 'max tokens', - minimum: 1, - maximum: 1_048_576, - ...(decl.defaults?.max_tokens !== undefined && { default: decl.defaults.max_tokens }), - }, - }, - additionalProperties: false, - }; -} - -export class ProviderRegistry { - private readonly iii: ISdk; - private readonly providers = new Map(); - private writeChain: Promise = Promise.resolve(); - - constructor(iii: ISdk) { - this.iii = iii; - } - - /** - * Ensure the `harness` entry exists with at least the permissions block, - * even before any provider declares. Tolerant of a missing configuration - * worker — logs and moves on. - */ - async init(): Promise { - await this.enqueue(() => this.ensureRegistered()); - } - - /** Record a provider declaration and re-register the composite schema. */ - async declare(decl: ProviderDeclaration): Promise { - if (!decl.id) throw new Error('provider declaration requires an id'); - await this.enqueue(async () => { - this.providers.set(decl.id, decl); - await this.ensureRegistered(); - }); - } - - list(): ProviderListEntry[] { - return [...this.providers.values()].map((d) => ({ - id: d.id, - display_name: d.display_name ?? d.id, - supports_model_listing: d.supports_model_listing ?? false, - })); - } - - /** Resolve a provider's credential + settings for a stream/complete call. */ - async resolve(providerId: string): Promise { - const value = await this.readValue({ raw: false }); - const cfg: HarnessProviderConfig = value.providers[providerId] ?? {}; - const decl = this.providers.get(providerId); - - let credential: Credential | null = null; - let source: 'stored' | 'environment' | null = null; - const storedKey = - typeof cfg.api_key === 'string' && cfg.api_key.length > 0 ? cfg.api_key : null; - if (storedKey) { - credential = { type: 'api_key', key: storedKey }; - source = 'stored'; - } else { - const envVar = decl?.credential_env_var; - const envKey = envVar ? process.env[envVar] : undefined; - if (envKey) { - credential = { type: 'api_key', key: envKey }; - source = 'environment'; - } - } - - const api_url = - typeof cfg.api_url === 'string' && cfg.api_url.length > 0 - ? cfg.api_url - : (decl?.defaults?.api_url ?? null); - // Only a user-configured max_tokens counts. The declared default is NOT - // seeded here — the per-model clamp (runtime/output-tokens.ts) would - // treat it as a deliberate override and pin every request to 8192; - // providers apply their own fallback after the clamp. - const max_tokens = - typeof cfg.max_tokens === 'number' && cfg.max_tokens > 0 ? cfg.max_tokens : null; - - return { configured: credential !== null, source, credential, api_url, max_tokens }; - } - - // ------------------------------------------------------------------------- - // Internals - // ------------------------------------------------------------------------- - - private composeSchema(): JsonSchema { - const providerProps: Record = {}; - for (const [id, decl] of this.providers) { - providerProps[id] = decl.config_schema ?? defaultProviderSchema(decl); - } - return { - $schema: 'http://json-schema.org/draft-07/schema#', - type: 'object', - title: ENTRY_NAME, - properties: { - permissions: { - type: 'object', - title: 'permissions', - properties: { - default_mode: { - type: 'string', - title: 'default mode', - description: 'Default approval mode applied to new agent sessions.', - enum: [...PERMISSION_MODES], - default: DEFAULT_PERMISSION_MODE, - }, - }, - required: ['default_mode'], - additionalProperties: false, - }, - providers: { - type: 'object', - title: 'providers', - description: 'Per-provider credentials and settings.', - properties: providerProps, - additionalProperties: false, - }, - }, - required: ['permissions', 'providers'], - additionalProperties: false, - }; - } - - private async readValue(opts: { raw: boolean }): Promise { - const raw = await configurationGet(this.iii, HARNESS_CONFIG_ID, { raw: opts.raw }); - return normalizeHarnessConfig(raw); - } - - private async ensureRegistered(): Promise { - const schema = this.composeSchema(); - try { - // Read the stored template form so re-registration preserves any - // operator-set value; only seed on the very first registration. - const existing = await configurationGet(this.iii, HARNESS_CONFIG_ID, { raw: true }); - await configurationRegister(this.iii, { - id: HARNESS_CONFIG_ID, - name: ENTRY_NAME, - description: ENTRY_DESCRIPTION, - schema, - ...(existing === null && { - initial_value: baseHarnessConfigValue() as unknown as JsonValue, - }), - }); - } catch (err) { - logger.warn('provider-registry: configuration::register failed', { err: String(err) }); - } - } - - private enqueue(op: () => Promise): Promise { - const next = this.writeChain.then(() => op()); - this.writeChain = next.then( - () => undefined, - () => undefined, - ); - return next; - } -} diff --git a/harness/src/harness/register.ts b/harness/src/harness/register.ts index 21dc334ef..5015257f6 100644 --- a/harness/src/harness/register.ts +++ b/harness/src/harness/register.ts @@ -4,10 +4,10 @@ import { register as registerTrigger } from './trigger.js'; import { loadHarnessConfig } from './config.js'; import { spawnPumps } from './fanout/index.js'; import { register as registerFs } from './fs.js'; +import { migrateLlmRouterConfig } from './migrate-llm-router-config.js'; +import { registerHarnessConfigEntry } from './permissions-config.js'; import { registerPolicy } from './policy/check-permissions.js'; import { loadAndWatch } from './policy/handle.js'; -import { registerProviderRegistry } from './providers/register.js'; -import { registerProviderRefreshOnConfig } from './providers/refresh-on-config.js'; import { FanoutState, registerSubscriptions } from './ui-subscribe.js'; export async function register(iii: ISdk, ctx: { configPath: string; url: string }): Promise { @@ -21,12 +21,14 @@ export async function register(iii: ISdk, ctx: { configPath: string; url: string spawnPumps(iii, fanoutState); registerFs(iii, ctx.url); - // Provider credentials + settings + permissions now live in the - // `configuration` worker (`harness` entry), owned by this registry. - const registry = await registerProviderRegistry(iii); - // Re-discover provider models whenever that entry changes so the picker - // reflects added/removed credentials without a manual refresh. - registerProviderRefreshOnConfig(iii, registry, fanoutState); + // Provider credentials/settings live in the router-owned `llm-router` + // entry; the `harness` entry keeps only the permissions block. Paste-a-key + // reactivity is the router's configuration trigger now. + await registerHarnessConfigEntry(iii); + // One-time copy of any pre-router provider config into the llm-router + // entry (internal retries — the router composes that entry's schema only + // after the first provider registers, so this must not block boot). + void migrateLlmRouterConfig(iii); const handle = await loadAndWatch(harness.permissions_path); registerPolicy(iii, handle); diff --git a/harness/src/index.ts b/harness/src/index.ts index c9721eed6..2abd37d14 100644 --- a/harness/src/index.ts +++ b/harness/src/index.ts @@ -15,12 +15,6 @@ import { register as registerContextCompaction } from './context-compaction/regi import { register as registerHarness } from './harness/register.js'; import { register as registerHookFanout } from './hook-fanout/register.js'; import { register as registerLlmBudget } from './llm-budget/register.js'; -import { register as registerModelsCatalog } from './models-catalog/register.js'; -import { register as registerProviderAnthropic } from './provider-anthropic/register.js'; -import { register as registerProviderKimi } from './provider-kimi/register.js'; -import { register as registerProviderLlamacpp } from './provider-llamacpp/register.js'; -import { register as registerProviderLmstudio } from './provider-lmstudio/register.js'; -import { register as registerProviderOpenai } from './provider-openai/register.js'; import { logger } from './runtime/otel.js'; import { DEFAULT_CONFIG_PATH, @@ -62,41 +56,6 @@ const WORKERS: readonly WorkerDefinition[] = [ 'Generic publish-collect primitive: publishes a topic via iii::durable::publish, collects subscriber replies on agent::hook_reply, applies a merge rule, returns the merged result.', register: (iii, ctx) => registerHookFanout(iii, ctx), }, - { - name: 'models-catalog', - description: 'Model capabilities catalog on the iii bus (models::list/get/supports/register).', - register: async (iii) => registerModelsCatalog(iii), - }, - { - name: 'provider-anthropic', - description: - 'Anthropic Messages API streaming provider on the iii bus (provider::anthropic::stream + ::complete).', - register: (iii, ctx) => registerProviderAnthropic(iii, ctx), - }, - { - name: 'provider-openai', - description: - 'OpenAI Chat Completions streaming provider on the iii bus (provider::openai::stream + ::complete).', - register: (iii, ctx) => registerProviderOpenai(iii, ctx), - }, - { - name: 'provider-kimi', - description: - 'Kimi (Moonshot) Chat Completions streaming provider on the iii bus (provider::kimi::stream + ::complete).', - register: (iii, ctx) => registerProviderKimi(iii, ctx), - }, - { - name: 'provider-lmstudio', - description: - 'LM Studio (localhost) Chat Completions streaming provider on the iii bus (provider::lmstudio::stream + ::complete).', - register: (iii, ctx) => registerProviderLmstudio(iii, ctx), - }, - { - name: 'provider-llamacpp', - description: - 'llama.cpp llama-server (localhost) Chat Completions streaming provider on the iii bus (provider::llamacpp::stream + ::complete).', - register: (iii, ctx) => registerProviderLlamacpp(iii, ctx), - }, { name: 'llm-budget', description: 'LLM spend caps with alerts, forecast, period rollover (budget::*).', diff --git a/harness/src/models-catalog/handlers/get.ts b/harness/src/models-catalog/handlers/get.ts deleted file mode 100644 index a2f835f40..000000000 --- a/harness/src/models-catalog/handlers/get.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { requireString } from '../../runtime/handler.js'; -import type { ISdk } from '../../runtime/iii.js'; -import { getFromState } from '../state.js'; - -export function register(iii: ISdk): void { - iii.registerFunction( - 'models::get', - async (payload: unknown) => { - const obj = (payload ?? {}) as Record; - const provider = requireString(obj, 'provider'); - const model_id = requireString(obj, 'model_id'); - return await getFromState(iii, provider, model_id); - }, - { - description: - 'Look up a single model by (provider, model_id). Returns null when no provider has registered it.', - }, - ); -} diff --git a/harness/src/models-catalog/handlers/list.ts b/harness/src/models-catalog/handlers/list.ts deleted file mode 100644 index ff2d26ffc..000000000 --- a/harness/src/models-catalog/handlers/list.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { ISdk } from '../../runtime/iii.js'; -import { listFromState } from '../state.js'; -import { parseCapability } from '../types.js'; - -export function register(iii: ISdk): void { - iii.registerFunction( - 'models::list', - async (payload: unknown) => { - const obj = (payload ?? {}) as Record; - const provider = typeof obj.provider === 'string' ? obj.provider : undefined; - const cap = typeof obj.capability === 'string' ? parseCapability(obj.capability) : null; - const models = await listFromState(iii, { - provider, - capability: cap ?? undefined, - }); - return { models }; - }, - { - description: - 'List models, optionally filtered by provider or capability. Returns only models registered by providers (no embedded seed).', - }, - ); -} diff --git a/harness/src/models-catalog/handlers/reconcile.ts b/harness/src/models-catalog/handlers/reconcile.ts deleted file mode 100644 index 4edea5a66..000000000 --- a/harness/src/models-catalog/handlers/reconcile.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { ISdk } from '../../runtime/iii.js'; -import { stateSet } from '../../runtime/state.js'; -import { isModel, providerStateKey } from '../state.js'; -import { MODELS_SCOPE, type Model } from '../types.js'; - -export function register(iii: ISdk): void { - iii.registerFunction( - 'models::reconcile', - async (payload: unknown) => { - const obj = (payload ?? {}) as Record; - const provider = typeof obj.provider === 'string' ? obj.provider : ''; - if (!provider) { - throw new Error('models::reconcile requires a provider'); - } - const raw = Array.isArray(obj.models) ? obj.models : []; - const models: Model[] = []; - for (const entry of raw) { - if (!isModel(entry)) continue; - if (entry.provider !== provider) { - throw new Error( - `models::reconcile: model ${entry.id} has provider ${entry.provider}, expected ${provider}`, - ); - } - models.push(entry); - } - await stateSet(iii, MODELS_SCOPE, providerStateKey(provider), models); - return { provider, count: models.length, ids: models.map((m) => m.id) }; - }, - { - description: - 'Replace the catalog for with a single Model[] value under scope models (one state write).', - }, - ); -} diff --git a/harness/src/models-catalog/handlers/supports.ts b/harness/src/models-catalog/handlers/supports.ts deleted file mode 100644 index 72ccb1467..000000000 --- a/harness/src/models-catalog/handlers/supports.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { requireString } from '../../runtime/handler.js'; -import type { ISdk } from '../../runtime/iii.js'; -import { getFromState } from '../state.js'; -import { parseCapability, supportsModel } from '../types.js'; - -export function register(iii: ISdk): void { - iii.registerFunction( - 'models::supports', - async (payload: unknown) => { - const obj = (payload ?? {}) as Record; - const provider = requireString(obj, 'provider'); - const model_id = requireString(obj, 'model_id'); - const capability = parseCapability(requireString(obj, 'capability')); - if (!capability) { - throw new Error('missing or unknown capability'); - } - const m = await getFromState(iii, provider, model_id); - return { supported: m ? supportsModel(m, capability) : false }; - }, - { - description: - 'Check whether a provider-registered model supports a capability (false when unknown).', - }, - ); -} diff --git a/harness/src/models-catalog/iii.worker.yaml b/harness/src/models-catalog/iii.worker.yaml deleted file mode 100644 index e8585a9c7..000000000 --- a/harness/src/models-catalog/iii.worker.yaml +++ /dev/null @@ -1,17 +0,0 @@ -iii: v1 -name: models-catalog -language: node -deploy: binary -manifest: package.json -bin: iii-models-catalog -description: Model capabilities knowledge base on the iii bus (models::*). State-backed catalog written by provider discovery via models::reconcile. - -runtime: - kind: node - -scripts: - install: pnpm install - start: node ./dist/models-catalog/main.js - -dependencies: - iii-state: "^0.11.0" diff --git a/harness/src/models-catalog/main.ts b/harness/src/models-catalog/main.ts deleted file mode 100644 index 4495b0e9a..000000000 --- a/harness/src/models-catalog/main.ts +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env node -import { bootstrapWorker } from '../runtime/worker.js'; -import { register } from './register.js'; - -await bootstrapWorker({ - name: 'models-catalog', - description: 'Model capabilities catalog on the iii bus (models::list/get/supports/reconcile).', - register: async (iii) => register(iii), -}); diff --git a/harness/src/models-catalog/register.ts b/harness/src/models-catalog/register.ts deleted file mode 100644 index 1cd55076a..000000000 --- a/harness/src/models-catalog/register.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { ISdk } from '../runtime/iii.js'; -import { register as registerGet } from './handlers/get.js'; -import { register as registerList } from './handlers/list.js'; -import { register as registerReconcile } from './handlers/reconcile.js'; -import { register as registerSupports } from './handlers/supports.js'; - -export function register(iii: ISdk): void { - registerList(iii); - registerGet(iii); - registerSupports(iii); - registerReconcile(iii); -} diff --git a/harness/src/models-catalog/state.ts b/harness/src/models-catalog/state.ts deleted file mode 100644 index 8cd06fc51..000000000 --- a/harness/src/models-catalog/state.ts +++ /dev/null @@ -1,40 +0,0 @@ -/** - * State-backed reads for the models catalog: scope `models`, one Model[] per - * provider key. `models::reconcile` validates entries via isModel before writing - * and state is cleared on deploy, so reads trust the stored shape (no re-parsing). - */ - -import type { ISdk } from '../runtime/iii.js'; -import { stateGet, stateListValues } from '../runtime/state.js'; -import { type ListFilter, MODELS_SCOPE, type Model, supportsModel } from './types.js'; - -export function providerStateKey(provider: string): string { - return provider; -} - -/** Write-side boundary guard for provider-discovery output (used by models::reconcile). */ -export function isModel(v: unknown): v is Model { - return Boolean(v && typeof v === 'object' && typeof (v as Model).id === 'string'); -} - -export async function getProviderModels(iii: ISdk, provider: string): Promise { - return (await stateGet(iii, MODELS_SCOPE, providerStateKey(provider))) ?? []; -} - -export async function listFromState(iii: ISdk, filter: ListFilter): Promise { - const { capability } = filter; - - if (filter.provider !== undefined) { - const models = await getProviderModels(iii, filter.provider); - return capability === undefined ? models : models.filter((m) => supportsModel(m, capability)); - } - - // Each provider key stores one Model[]; flatten across providers. - const out = (await stateListValues(iii, { scope: MODELS_SCOPE })).flat(); - return capability === undefined ? out : out.filter((m) => supportsModel(m, capability)); -} - -export async function getFromState(iii: ISdk, provider: string, id: string): Promise { - const models = await getProviderModels(iii, provider); - return models.find((m) => m.id === id) ?? null; -} diff --git a/harness/src/models-catalog/types.ts b/harness/src/models-catalog/types.ts deleted file mode 100644 index 14bf579f1..000000000 --- a/harness/src/models-catalog/types.ts +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Model capability types. Wire-identical to - * `models-catalog/src/lib.rs::{Model, Pricing, …}`. - */ - -export type ThinkingLevel = 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; - -export type Transport = 'sse' | 'websocket' | 'auto'; - -export type CacheRetention = 'none' | 'short' | 'long'; - -export type ThinkingBudgets = { - minimal?: number; - low?: number; - medium?: number; - high?: number; -}; - -export type Pricing = { - input_per_1m: number; - output_per_1m: number; - cache_read_per_1m?: number; - cache_write_per_1m?: number; -}; - -export type Model = { - id: string; - provider: string; - api: string; - display_name: string; - context_window: number; - max_output_tokens?: number; - supports_thinking?: boolean; - supports_xhigh?: boolean; - supports_tools?: boolean; - supports_vision?: boolean; - supports_cache?: boolean; - thinking_budgets?: ThinkingBudgets; - transports?: Transport[]; - default_cache_retention?: CacheRetention; - pricing?: Pricing; -}; - -export const MODELS_SCOPE = 'models'; - -export type Capability = - | { type: 'thinking' } - | { type: 'thinking_level'; level: ThinkingLevel } - | { type: 'tools' } - | { type: 'vision' } - | { type: 'cache' }; - -/** Filter for `models::list`, applied against the provider-registered catalog. */ -export type ListFilter = { - provider?: string; - capability?: Capability; -}; - -export function parseCapability(s: string): Capability | null { - switch (s) { - case 'thinking': - return { type: 'thinking' }; - case 'thinking:low': - return { type: 'thinking_level', level: 'low' }; - case 'thinking:medium': - return { type: 'thinking_level', level: 'medium' }; - case 'thinking:high': - return { type: 'thinking_level', level: 'high' }; - case 'thinking:xhigh': - return { type: 'thinking_level', level: 'xhigh' }; - case 'tools': - return { type: 'tools' }; - case 'vision': - return { type: 'vision' }; - case 'cache': - return { type: 'cache' }; - default: - return null; - } -} - -export function supportsModel(m: Model, capability: Capability): boolean { - switch (capability.type) { - case 'thinking': - return Boolean(m.supports_thinking); - case 'thinking_level': - return capability.level === 'xhigh' - ? Boolean(m.supports_xhigh) - : Boolean(m.supports_thinking); - case 'tools': - return Boolean(m.supports_tools); - case 'vision': - return Boolean(m.supports_vision); - case 'cache': - return Boolean(m.supports_cache); - } -} diff --git a/harness/src/provider-anthropic/auth.ts b/harness/src/provider-anthropic/auth.ts deleted file mode 100644 index f57d94a74..000000000 --- a/harness/src/provider-anthropic/auth.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Resolve the Anthropic credential + runtime settings from the harness - * provider registry (`harness::provider::resolve`), then turn them into an - * AnthropicConfig. Replaces the old `auth::get_token` + `provider_config::get` - * pair with a single call. - */ - -import type { Model } from '../models-catalog/types.js'; -import type { ISdk } from '../runtime/iii.js'; -import { clampOutputTokens, getCatalogModel } from '../runtime/output-tokens.js'; -import { type ProviderResolveResult, resolveProvider } from '../runtime/provider-resolve.js'; -import type { WorkerConfig } from './config.js'; -import { type AnthropicConfig, configWithCredential } from './types.js'; - -export const PROVIDER_ID = 'anthropic'; - -/** - * Single-slot cache of the resolved provider credential, keyed by the turn's - * stable id. The credential is global, so within a turn its 20+ stream calls - * reuse one resolution; a new turn (new key) re-resolves, picking up a key - * rotated between turns. {@link invalidateProviderResolveCache} drops it on a - * mid-turn 401. With no key threaded, callers always resolve (no caching). - */ -let resolveCache: { key: number; resolved: ProviderResolveResult } | null = null; - -async function resolveProviderForTurn(iii: ISdk, key?: number): Promise { - if (key === undefined) return resolveProvider(iii, PROVIDER_ID); - if (resolveCache?.key === key) return resolveCache.resolved; - const resolved = await resolveProvider(iii, PROVIDER_ID); - resolveCache = { key, resolved }; - return resolved; -} - -/** Drop the cached resolution so the next stream re-resolves (called on a 401). */ -export function invalidateProviderResolveCache(): void { - resolveCache = null; -} - -/** Test seam: clear the cache between cases. */ -export function _resetProviderResolveCacheForTests(): void { - resolveCache = null; -} - -export async function buildConfig( - iii: ISdk, - worker: WorkerConfig, - model: string, - preResolved?: Model, - resolutionKey?: number, -): Promise { - const resolved = await resolveProviderForTurn(iii, resolutionKey); - if (!resolved.credential) { - throw new Error( - 'harness::provider::resolve returned no credential for provider `anthropic` ' + - '(set an api key in the harness configuration or ANTHROPIC_API_KEY)', - ); - } - const apiUrl = resolved.api_url ?? worker.default_api_url; - const catalog = preResolved ?? (await getCatalogModel(iii, PROVIDER_ID, model)); - const maxTokens = clampOutputTokens({ - modelMaxOutput: catalog?.max_output_tokens, - userOverride: resolved.max_tokens, - workerDefault: worker.default_max_tokens, - }); - const cfg = configWithCredential(model, resolved.credential, maxTokens, apiUrl); - return catalog ? { ...cfg, catalog } : cfg; -} diff --git a/harness/src/provider-anthropic/cache.ts b/harness/src/provider-anthropic/cache.ts deleted file mode 100644 index eb8d49389..000000000 --- a/harness/src/provider-anthropic/cache.ts +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Anthropic prompt-cache markers. Mirrors - * `provider-anthropic/src/lib.rs::{build_system_field, - * apply_tools_cache_control, apply_messages_cache_anchor}`. - */ - -const CACHE_MIN_CHARS = 4096; -const CACHE_FLAG_ENV = 'HARNESS_ANTHROPIC_CACHE'; - -let cacheEnabledCached: boolean | null = null; -function cacheEnabled(): boolean { - if (cacheEnabledCached !== null) return cacheEnabledCached; - const v = process.env[CACHE_FLAG_ENV]; - cacheEnabledCached = v === undefined || !['0', 'false', 'FALSE', 'False'].includes(v); - return cacheEnabledCached; -} - -const ephemeral = () => ({ type: 'ephemeral' }); - -export function buildSystemField(prompt: string): unknown { - if (cacheEnabled() && prompt.length >= CACHE_MIN_CHARS) { - return [{ type: 'text', text: prompt, cache_control: ephemeral() }]; - } - return prompt; -} - -export function applyToolsCacheControl(tools: Record[]): void { - if (!cacheEnabled() || tools.length === 0) return; - const size = tools.reduce((acc, t) => acc + JSON.stringify(t).length, 0); - if (size < CACHE_MIN_CHARS) return; - const last = tools[tools.length - 1]; - if (last) last.cache_control = ephemeral(); -} - -export function applyMessagesCacheAnchor(wire: Record[]): void { - if (!cacheEnabled() || wire.length === 0) return; - let lastStable = -1; - for (let i = wire.length - 1; i >= 0; i--) { - if (isStableAssistant(wire, i)) { - lastStable = i; - break; - } - } - if (lastStable < 0) return; - const msg = wire[lastStable]; - if (!msg) return; - const content = msg.content; - if (!Array.isArray(content) || content.length === 0) return; - // Anchor on the last block that accepts cache_control — Anthropic rejects - // it on thinking/redacted_thinking blocks, which can trail a turn under - // interleaved thinking. - for (let i = content.length - 1; i >= 0; i--) { - const block = content[i] as Record | undefined; - if (!block) continue; - if (block.type === 'thinking' || block.type === 'redacted_thinking') continue; - block.cache_control = ephemeral(); - return; - } -} - -function isStableAssistant(wire: Record[], idx: number): boolean { - const msg = wire[idx]; - if (!msg) return false; - if (msg.role !== 'assistant') return false; - const content = msg.content; - if (!Array.isArray(content)) return true; - const toolUseIds = content - .filter( - (b): b is { id: string } => - Boolean(b) && - typeof b === 'object' && - (b as Record).type === 'tool_use' && - typeof (b as Record).id === 'string', - ) - .map((b) => b.id); - if (toolUseIds.length === 0) return true; - return toolUseIds.every((id) => hasDownstreamToolResult(wire.slice(idx + 1), id)); -} - -function hasDownstreamToolResult(later: Record[], id: string): boolean { - return later.some((m) => { - if (m.role !== 'user') return false; - const content = m.content; - if (!Array.isArray(content)) return false; - return content.some( - (b) => - b && - typeof b === 'object' && - (b as Record).type === 'tool_result' && - (b as Record).tool_use_id === id, - ); - }); -} diff --git a/harness/src/provider-anthropic/complete.ts b/harness/src/provider-anthropic/complete.ts deleted file mode 100644 index b78649bd0..000000000 --- a/harness/src/provider-anthropic/complete.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Legacy `provider::anthropic::complete`. Drains the stream and returns - * the final AssistantMessage. Used by the Phase 1 `router::stream_assistant` - * code path; left in place for back-compat. - */ - -import { requireString } from '../runtime/handler.js'; -import type { ISdk } from '../runtime/iii.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { collect, streamAnthropic } from './stream.js'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - 'provider::anthropic::complete', - async (payload: unknown) => { - const obj = (payload ?? {}) as Record; - const model = requireString(obj, 'model'); - const system_prompt = typeof obj.system_prompt === 'string' ? obj.system_prompt : ''; - const messages = Array.isArray(obj.messages) ? (obj.messages as AgentMessage[]) : []; - const tools = Array.isArray(obj.tools) ? (obj.tools as AgentFunction[]) : []; - const cfg = await buildConfig(iii, worker, model); - const events = streamAnthropic({ cfg, system_prompt, messages, tools }); - return await collect(events); - }, - { - description: - 'Legacy: drain a streamed Anthropic completion and return the final AssistantMessage.', - }, - ); -} diff --git a/harness/src/provider-anthropic/config.ts b/harness/src/provider-anthropic/config.ts deleted file mode 100644 index 55b24bb4b..000000000 --- a/harness/src/provider-anthropic/config.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { getNumber, getSection, getString } from '../runtime/config.js'; - -export type WorkerConfig = { - default_max_tokens: number; - default_api_url: string; -}; - -export const DEFAULT_API_URL = 'https://api.anthropic.com/v1/messages'; - -export function loadWorkerConfig(cfg: Record): WorkerConfig { - const section = getSection(cfg, 'provider_anthropic'); - return { - default_max_tokens: getNumber(section, 'default_max_tokens', 8192), - default_api_url: getString(section, 'default_api_url', DEFAULT_API_URL), - }; -} diff --git a/harness/src/provider-anthropic/discover.ts b/harness/src/provider-anthropic/discover.ts deleted file mode 100644 index 036ba4ae2..000000000 --- a/harness/src/provider-anthropic/discover.ts +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Anthropic model discovery — hits `GET /v1/models` and registers each - * returned model into the iii models catalog so the picker shows the live - * list (cached), with a default context window + conservative capability - * flags (upstream `/v1/models` exposes little metadata). - * - * Best-effort: a missing credential or any upstream error yields `[]`. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { - deriveModelsUrl, - enrichModel, - fetchModelsForDiscovery, - type ModelStub, - reconcileModels, -} from '../runtime/models-discovery.js'; -import { getModelsDevIndex, lookupModelsDev } from '../runtime/modelsdev.js'; -import { logger } from '../runtime/otel.js'; -import { resolveProvider } from '../runtime/provider-resolve.js'; -import { PROVIDER_ID } from './auth.js'; -import type { WorkerConfig } from './config.js'; - -const ANTHROPIC_VERSION = '2023-06-01'; -const DEFAULT_CONTEXT_WINDOW = 200_000; - -type AnthropicModel = { id?: unknown; display_name?: unknown }; - -function parseStubs(json: unknown): ModelStub[] { - const data = (json as { data?: unknown })?.data; - if (!Array.isArray(data)) return []; - const out: ModelStub[] = []; - for (const raw of data as AnthropicModel[]) { - const id = typeof raw.id === 'string' && raw.id.length > 0 ? raw.id : null; - if (!id) continue; - out.push({ - id, - display_name: typeof raw.display_name === 'string' ? raw.display_name : undefined, - }); - } - return out; -} - -export async function discoverAndRegister(iii: ISdk, worker: WorkerConfig): Promise { - const resolved = await resolveProvider(iii, PROVIDER_ID).catch(() => null); - const cred = resolved?.credential ?? null; - if (!cred) { - // No credential: drop any models a previous run registered so the picker - // reflects the removal instead of showing stale, unusable rows. - logger.info('anthropic discovery: no credential; pruning catalog', {}); - await reconcileModels(iii, PROVIDER_ID, []); - return []; - } - const key = cred.type === 'api_key' ? cred.key : cred.access_token; - const url = deriveModelsUrl(resolved?.api_url ?? worker.default_api_url); - const fetchResult = await fetchModelsForDiscovery(url, { - 'x-api-key': key, - 'anthropic-version': ANTHROPIC_VERSION, - }); - if (fetchResult.kind === 'auth_error') { - logger.info('anthropic discovery: invalid credential; pruning catalog', { - status: fetchResult.status, - }); - await reconcileModels(iii, PROVIDER_ID, []); - return []; - } - if (fetchResult.kind !== 'ok') return []; - - const modelsDev = await getModelsDevIndex(); - const models = parseStubs(fetchResult.json).map((stub) => - enrichModel({ - provider: PROVIDER_ID, - api: 'anthropic-messages', - stub, - defaultContextWindow: DEFAULT_CONTEXT_WINDOW, - modelsDev: lookupModelsDev(modelsDev, PROVIDER_ID, stub.id), - }), - ); - const registered = await reconcileModels(iii, PROVIDER_ID, models); - logger.info('anthropic discovery: reconciled models', { - count: registered.length, - discovered: models.length, - }); - return registered; -} diff --git a/harness/src/provider-anthropic/iii.worker.yaml b/harness/src/provider-anthropic/iii.worker.yaml deleted file mode 100644 index 704376eb2..000000000 --- a/harness/src/provider-anthropic/iii.worker.yaml +++ /dev/null @@ -1,17 +0,0 @@ -iii: v1 -name: provider-anthropic -language: node -deploy: binary -manifest: package.json -bin: iii-provider-anthropic -description: Anthropic Messages API streaming provider; exposes provider::anthropic::stream and provider::anthropic::complete on the iii bus. - -runtime: - kind: node - -scripts: - install: pnpm install - start: node ./dist/provider-anthropic/main.js --config ./config.yaml - -dependencies: - configuration: "^0.11.0" diff --git a/harness/src/provider-anthropic/main.ts b/harness/src/provider-anthropic/main.ts deleted file mode 100644 index 19ed47013..000000000 --- a/harness/src/provider-anthropic/main.ts +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env node -import { bootstrapWorker } from '../runtime/worker.js'; -import { register } from './register.js'; - -await bootstrapWorker({ - name: 'provider-anthropic', - description: - 'Anthropic Messages API streaming provider on the iii bus (provider::anthropic::stream + ::complete).', - register: (iii, ctx) => register(iii, ctx), -}); diff --git a/harness/src/provider-anthropic/refresh-fn.ts b/harness/src/provider-anthropic/refresh-fn.ts deleted file mode 100644 index 1eb38b22a..000000000 --- a/harness/src/provider-anthropic/refresh-fn.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * `provider::anthropic::refresh_models` — re-pull the upstream model list - * and register each into the iii models catalog. Returns `{ registered }`. - * Never throws across the bus boundary. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import type { WorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; - -export const FUNCTION_ID = 'provider::anthropic::refresh_models'; - -export type RefreshResult = { registered: string[] }; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (): Promise => { - try { - return { registered: await discoverAndRegister(iii, worker) }; - } catch (err) { - logger.warn('provider::anthropic::refresh_models failed', { err: String(err) }); - return { registered: [] }; - } - }, - { - description: - 'Re-pull the Anthropic model list (GET /v1/models) and register each into the iii models catalog. Idempotent.', - }, - ); -} diff --git a/harness/src/provider-anthropic/register.ts b/harness/src/provider-anthropic/register.ts deleted file mode 100644 index 4c8ade32d..000000000 --- a/harness/src/provider-anthropic/register.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { loadConfig } from '../runtime/config.js'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import { declareProvider } from '../runtime/provider-resolve.js'; -import { PROVIDER_ID } from './auth.js'; -import { register as registerComplete } from './complete.js'; -import { loadWorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; -import { register as registerRefresh } from './refresh-fn.js'; -import { register as registerStream } from './stream-fn.js'; - -export async function register(iii: ISdk, ctx: { configPath: string }): Promise { - const cfg = await loadConfig(ctx.configPath); - const worker = loadWorkerConfig(cfg); - registerComplete(iii, worker); - registerStream(iii, worker); - registerRefresh(iii, worker); - - // Self-declare into the harness configuration schema (api key + settings). - void declareProvider(iii, { - id: PROVIDER_ID, - display_name: 'anthropic', - credential_env_var: 'ANTHROPIC_API_KEY', - defaults: { - api_url: worker.default_api_url, - max_tokens: worker.default_max_tokens, - }, - supports_model_listing: true, - }); - - // Fire-and-forget startup discovery: pull the live model list into the - // catalog so the picker shows current models. Deferred so a slow upstream - // (or the registry still coming up) doesn't block harness boot. - setImmediate(() => { - discoverAndRegister(iii, worker).catch((err) => { - logger.warn('anthropic startup discovery threw', { err: String(err) }); - }); - }); -} diff --git a/harness/src/provider-anthropic/sse.ts b/harness/src/provider-anthropic/sse.ts deleted file mode 100644 index 0051ede61..000000000 --- a/harness/src/provider-anthropic/sse.ts +++ /dev/null @@ -1,312 +0,0 @@ -/** - * Anthropic SSE stream parser + state machine. Mirrors - * `provider-anthropic/src/lib.rs::{handle_sse_event, build_partial, - * build_final, build_content, merge_usage, map_stop_reason}` and the - * surrounding stream loop. - * - * Consumes `data: {…}` SSE frames, threads them through the partial - * state, and yields `AssistantMessageEvent` values for the caller. - */ - -import { logger } from '../runtime/otel.js'; -import { type AssistantMessage, emptyAssistant } from '../types/agent-message.js'; -import type { ContentBlock, TextContent, ThinkingContent } from '../types/content.js'; -import type { AssistantMessageEvent, ErrorKind, StopReason, Usage } from '../types/stream-event.js'; -import { decodeToolName } from './wire-messages.js'; - -type PartialFunctionCall = { id: string; function_id: string; args_json: string }; - -type PartialThinking = { text: string; signature?: string }; - -type BlockKind = 'text' | 'tool_use' | 'thinking'; - -type OpenBlockKind = BlockKind | null; - -export type PartialState = { - text_blocks: string[]; - thinking_blocks: PartialThinking[]; - function_calls: PartialFunctionCall[]; - /** - * Wire arrival order of content blocks ({kind, index-within-kind-array}). - * Replayed turns must keep thinking blocks in their original position - * relative to tool_use, so `buildContent` reconstructs in this order. - */ - block_order: Array<{ kind: BlockKind; idx: number }>; - /** Kind of the currently open content block so `content_block_stop` emits the matching end event. */ - open_block: OpenBlockKind; - usage: Usage; - stop_reason: StopReason; - error_message: string | null; -}; - -export function emptyPartial(): PartialState { - return { - text_blocks: [], - thinking_blocks: [], - function_calls: [], - block_order: [], - open_block: null, - usage: { input: 0, output: 0, cache_read: 0, cache_write: 0 }, - stop_reason: 'end', - error_message: null, - }; -} - -function pushBlockContent(out: ContentBlock[], state: PartialState, kind: BlockKind, idx: number) { - if (kind === 'thinking') { - const th = state.thinking_blocks[idx]; - if (th && th.text.length > 0) { - const tc: ThinkingContent = { type: 'thinking', text: th.text }; - if (th.signature) tc.signature = th.signature; - out.push(tc); - } - return; - } - if (kind === 'text') { - const t = state.text_blocks[idx]; - if (t !== undefined && t.length > 0) { - const tc: TextContent = { type: 'text', text: t }; - out.push(tc); - } - return; - } - const tc = state.function_calls[idx]; - if (!tc) return; - let args: unknown = {}; - if (tc.args_json.length > 0) { - try { - args = JSON.parse(tc.args_json); - } catch { - args = null; - } - } - out.push({ - type: 'function_call', - id: tc.id, - function_id: tc.function_id, - arguments: args, - }); -} - -function buildContent(state: PartialState): ContentBlock[] { - const out: ContentBlock[] = []; - // Wire arrival order first; block indices not tracked in block_order - // (state built directly in tests) are appended grouped afterwards. - const seen = { - text: new Set(), - thinking: new Set(), - tool_use: new Set(), - }; - for (const e of state.block_order) { - if (seen[e.kind].has(e.idx)) continue; - seen[e.kind].add(e.idx); - pushBlockContent(out, state, e.kind, e.idx); - } - for (let i = 0; i < state.thinking_blocks.length; i++) { - if (!seen.thinking.has(i)) pushBlockContent(out, state, 'thinking', i); - } - for (let i = 0; i < state.text_blocks.length; i++) { - if (!seen.text.has(i)) pushBlockContent(out, state, 'text', i); - } - for (let i = 0; i < state.function_calls.length; i++) { - if (!seen.tool_use.has(i)) pushBlockContent(out, state, 'tool_use', i); - } - return out; -} - -export function buildPartial(state: PartialState, model: string): AssistantMessage { - return { - role: 'assistant', - content: buildContent(state), - stop_reason: state.stop_reason, - error_message: state.error_message, - error_kind: null, - usage: state.usage, - model, - provider: 'anthropic', - timestamp: Date.now(), - }; -} - -export function buildFinal(state: PartialState, model: string): AssistantMessage { - return buildPartial(state, model); -} - -export function mapStopReason(s: string): StopReason { - switch (s) { - case 'end_turn': - return 'end'; - case 'max_tokens': - return 'length'; - case 'tool_use': - return 'function_call'; - case 'stop_sequence': - return 'end'; - default: - return 'end'; - } -} - -export function mergeUsage(usage: Record, into: Usage): void { - const num = (k: string) => (typeof usage[k] === 'number' ? (usage[k] as number) : 0); - into.input = (into.input ?? 0) + num('input_tokens'); - into.output = (into.output ?? 0) + num('output_tokens'); - into.cache_read = (into.cache_read ?? 0) + num('cache_read_input_tokens'); - into.cache_write = (into.cache_write ?? 0) + num('cache_creation_input_tokens'); -} - -/** Process a single SSE event block into 0+ AssistantMessageEvents. */ -export function handleSseEvent( - block: string, - state: PartialState, - model: string, -): AssistantMessageEvent[] { - let dataLine: string | null = null; - for (const line of block.split('\n')) { - if (line.startsWith('data: ')) dataLine = line.slice('data: '.length); - } - if (!dataLine) return []; - let parsed: Record; - try { - parsed = JSON.parse(dataLine) as Record; - } catch { - return []; - } - const eventType = typeof parsed.type === 'string' ? parsed.type : null; - if (!eventType) return []; - const events: AssistantMessageEvent[] = []; - switch (eventType) { - case 'message_start': { - const u = (parsed.message as Record | undefined)?.usage; - if (u && typeof u === 'object') mergeUsage(u as Record, state.usage); - break; - } - case 'content_block_start': { - const cb = parsed.content_block as Record | undefined; - const blockType = typeof cb?.type === 'string' ? cb.type : ''; - if (blockType === 'text') { - state.block_order.push({ kind: 'text', idx: state.text_blocks.length }); - state.text_blocks.push(''); - state.open_block = 'text'; - events.push({ type: 'text_start', partial: buildPartial(state, model) }); - } else if (blockType === 'tool_use') { - const id = typeof cb?.id === 'string' ? cb.id : ''; - const name = typeof cb?.name === 'string' ? decodeToolName(cb.name) : ''; - state.block_order.push({ kind: 'tool_use', idx: state.function_calls.length }); - state.function_calls.push({ id, function_id: name, args_json: '' }); - state.open_block = 'tool_use'; - events.push({ - type: 'functioncall_start', - partial: buildPartial(state, model), - }); - } else if (blockType === 'thinking' || blockType === 'redacted_thinking') { - // Redacted thinking is opaque and not persisted/round-tripped - // (needs a ContentBlock extension — follow-up); logged because the - // API expects it back during tool use. - if (blockType === 'redacted_thinking') { - logger.warn('anthropic redacted_thinking block received; not persisted/round-tripped', { - model, - }); - } - state.block_order.push({ kind: 'thinking', idx: state.thinking_blocks.length }); - state.thinking_blocks.push({ text: '' }); - state.open_block = 'thinking'; - events.push({ type: 'thinking_start', partial: buildPartial(state, model) }); - } - break; - } - case 'content_block_delta': { - const delta = parsed.delta as Record | undefined; - const dt = typeof delta?.type === 'string' ? delta.type : ''; - if (dt === 'text_delta') { - const text = typeof delta?.text === 'string' ? delta.text : ''; - const last = state.text_blocks[state.text_blocks.length - 1]; - if (last !== undefined) { - state.text_blocks[state.text_blocks.length - 1] = last + text; - } - events.push({ - type: 'text_delta', - partial: buildPartial(state, model), - delta: text, - }); - } else if (dt === 'input_json_delta') { - const json = typeof delta?.partial_json === 'string' ? delta.partial_json : ''; - const last = state.function_calls[state.function_calls.length - 1]; - if (last) last.args_json += json; - events.push({ - type: 'functioncall_delta', - partial: buildPartial(state, model), - delta: json, - }); - } else if (dt === 'thinking_delta') { - const text = typeof delta?.thinking === 'string' ? delta.thinking : ''; - const last = state.thinking_blocks[state.thinking_blocks.length - 1]; - if (last) last.text += text; - events.push({ - type: 'thinking_delta', - partial: buildPartial(state, model), - delta: text, - }); - } else if (dt === 'signature_delta') { - const sig = typeof delta?.signature === 'string' ? delta.signature : ''; - const last = state.thinking_blocks[state.thinking_blocks.length - 1]; - if (last && sig) last.signature = (last.signature ?? '') + sig; - } - break; - } - case 'content_block_stop': { - // Emit the end event matching the open block. Default to text_end for - // unknown/untracked blocks (preserves pre-thinking behavior). - const kind = state.open_block; - state.open_block = null; - if (kind === 'thinking') { - events.push({ type: 'thinking_end', partial: buildPartial(state, model) }); - } else if (kind === 'tool_use') { - events.push({ type: 'functioncall_end', partial: buildPartial(state, model) }); - } else { - events.push({ type: 'text_end', partial: buildPartial(state, model) }); - } - break; - } - case 'message_delta': { - const d = parsed.delta as Record | undefined; - const sr = typeof d?.stop_reason === 'string' ? d.stop_reason : null; - if (sr) state.stop_reason = mapStopReason(sr); - const u = parsed.usage as Record | undefined; - if (u) mergeUsage(u, state.usage); - break; - } - case 'message_stop': { - events.push({ - type: 'stop', - stop_reason: state.stop_reason, - error_message: state.error_message ?? undefined, - }); - break; - } - } - return events; -} - -export function syntheticErrorEvent( - message: string, - model: string, - error_kind: ErrorKind = 'transient', -): AssistantMessageEvent { - const final: AssistantMessage = { - ...emptyAssistant('anthropic', model), - content: [{ type: 'text', text: message }], - stop_reason: 'error', - error_message: message, - error_kind, - }; - return { type: 'error', error: final }; -} - -export function classifyAnthropicError(message: string, status?: number): ErrorKind { - if (status === 401 || status === 403) return 'auth_expired'; - if (status === 429) return 'rate_limited'; - if (status && status >= 500) return 'transient'; - if (/context|too large|too many tokens/i.test(message)) return 'context_overflow'; - return 'permanent'; -} diff --git a/harness/src/provider-anthropic/stream-fn.ts b/harness/src/provider-anthropic/stream-fn.ts deleted file mode 100644 index 78137092e..000000000 --- a/harness/src/provider-anthropic/stream-fn.ts +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Phase 2.A: `provider::anthropic::stream`. Takes a `ProviderStreamInput` - * (with `writer_ref`), pushes each AssistantMessageEvent as a JSON text - * message, closes the channel, returns `ProviderStreamOutput`. - * - * Mirrors the contract spelled out in `PHASE-2-PLAN.md` §4. - */ - -import type { ChannelWriter } from 'iii-sdk'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import { - ProviderStreamInputJsonSchema, - ProviderStreamOutputJsonSchema, - ProviderStreamRuntimeInputSchema, -} from '../types/provider.js'; -import type { AssistantMessageEvent } from '../types/stream-event.js'; -import { isTerminal } from '../types/stream-event.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { streamAnthropic } from './stream.js'; -import { buildThinkingConfig } from './thinking.js'; - -export const FUNCTION_ID = 'provider::anthropic::stream'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (raw: unknown) => { - const input = ProviderStreamRuntimeInputSchema.parse(raw); - const writer = input.writer_ref as ChannelWriter; - const cfg = await buildConfig( - iii, - worker, - input.model, - input.model_meta, - input.resolution_key, - ); - const thinking = buildThinkingConfig(input.thinking_level, cfg.max_tokens, cfg.catalog); - try { - const events = streamAnthropic({ - cfg, - system_prompt: input.system_prompt ?? '', - messages: input.messages as AgentMessage[], - tools: input.tools as import('../types/function.js').AgentFunction[], - ...(thinking ? { thinking } : {}), - }); - for await (const ev of events) { - writer.sendMessage(JSON.stringify(ev)); - if (isTerminal(ev as AssistantMessageEvent)) break; - } - } catch (err) { - logger.warn('provider::anthropic::stream failed mid-flight', { - err: String(err), - }); - } finally { - try { - writer.close(); - } catch (err) { - logger.debug('writer.close failed', { err: String(err) }); - } - } - return { ok: true }; - }, - { - description: - 'Stream a single assistant turn from Anthropic into the caller-supplied channel. Each AssistantMessageEvent is sent as a JSON text message; the terminal event is Done or Error followed by close.', - request_format: ProviderStreamInputJsonSchema as Record, - response_format: ProviderStreamOutputJsonSchema as Record, - }, - ); -} diff --git a/harness/src/provider-anthropic/stream.ts b/harness/src/provider-anthropic/stream.ts deleted file mode 100644 index 7ad5cc38b..000000000 --- a/harness/src/provider-anthropic/stream.ts +++ /dev/null @@ -1,152 +0,0 @@ -/** - * `streamAnthropic` — POST `/v1/messages` (stream:true), parse SSE, - * yield `AssistantMessageEvent`s. Mirrors - * `provider-anthropic/src/lib.rs::stream` + `stream_inner`. - */ - -import { logger } from '../runtime/otel.js'; -import type { AgentMessage, AssistantMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import type { AssistantMessageEvent } from '../types/stream-event.js'; -import { invalidateProviderResolveCache } from './auth.js'; -import { applyMessagesCacheAnchor, applyToolsCacheControl, buildSystemField } from './cache.js'; -import { - buildFinal, - classifyAnthropicError, - emptyPartial, - handleSseEvent, - syntheticErrorEvent, -} from './sse.js'; -import type { ThinkingConfig } from './thinking.js'; -import { type AnthropicConfig, authHeaderFor } from './types.js'; -import { toWireMessages } from './wire-messages.js'; -import { functionsToWire } from './wire-tools.js'; - -export type StreamArgs = { - cfg: AnthropicConfig; - system_prompt: string; - messages: AgentMessage[]; - tools: AgentFunction[]; - /** Extended thinking; absent = off. See `thinking.ts`. */ - thinking?: ThinkingConfig; -}; - -export async function* streamAnthropic({ - cfg, - system_prompt, - messages, - tools, - thinking, -}: StreamArgs): AsyncGenerator { - const wire_messages = toWireMessages(messages) as Record[]; - applyMessagesCacheAnchor(wire_messages); - const wire_tools = functionsToWire(tools) as Record[]; - applyToolsCacheControl(wire_tools); - const body = { - model: cfg.model, - max_tokens: cfg.max_tokens, - ...(thinking ? { thinking } : {}), - system: buildSystemField(system_prompt), - messages: wire_messages, - tools: wire_tools, - stream: true, - }; - const [headerName, headerValue] = authHeaderFor(cfg); - const headers: Record = { - [headerName]: headerValue, - 'anthropic-version': '2023-06-01', - 'content-type': 'application/json', - }; - if (thinking) { - headers['anthropic-beta'] = 'interleaved-thinking-2025-05-14'; - } - - const ac = new AbortController(); - let resp: Response; - try { - resp = await fetch(cfg.api_url, { - method: 'POST', - headers, - body: JSON.stringify(body), - signal: ac.signal, - }); - } catch (err) { - yield syntheticErrorEvent(`anthropic fetch failed: ${String(err)}`, cfg.model); - return; - } - - if (!resp.ok) { - const text = await resp.text().catch(() => ''); - const kind = classifyAnthropicError(text, resp.status); - if (kind === 'auth_expired') invalidateProviderResolveCache(); - yield syntheticErrorEvent(text || `anthropic http ${resp.status}`, cfg.model, kind); - return; - } - - const partialMsg: AssistantMessage = { - role: 'assistant', - content: [], - stop_reason: 'end', - error_message: null, - error_kind: null, - usage: null, - model: cfg.model, - provider: 'anthropic', - timestamp: Date.now(), - }; - yield { type: 'start', partial: partialMsg }; - - const state = emptyPartial(); - let buf = ''; - if (!resp.body) { - yield syntheticErrorEvent('anthropic response missing body', cfg.model); - return; - } - const reader = resp.body.getReader(); - const decoder = new TextDecoder(); - try { - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - let idx = buf.indexOf('\n\n'); - while (idx >= 0) { - const block = buf.slice(0, idx); - buf = buf.slice(idx + 2); - const events = handleSseEvent(block, state, cfg.model); - for (const e of events) yield e; - idx = buf.indexOf('\n\n'); - } - } - } catch (err) { - logger.warn('anthropic stream read failed', { err: String(err) }); - yield syntheticErrorEvent(`stream read failed: ${String(err)}`, cfg.model); - return; - } - yield { type: 'done', message: buildFinal(state, cfg.model) }; -} - -/** Drain a stream into the final AssistantMessage. */ -export async function collect( - events: AsyncIterable, -): Promise { - let last: AssistantMessage | null = null; - for await (const ev of events) { - if (ev.type === 'done') return ev.message; - if (ev.type === 'error') return ev.error; - if ('partial' in ev) last = ev.partial; - } - return ( - last ?? { - role: 'assistant', - content: [], - stop_reason: 'error', - error_message: 'stream closed without final', - error_kind: 'transient', - usage: null, - model: 'anthropic', - provider: 'anthropic', - timestamp: Date.now(), - } - ); -} diff --git a/harness/src/provider-anthropic/thinking.ts b/harness/src/provider-anthropic/thinking.ts deleted file mode 100644 index 08e349149..000000000 --- a/harness/src/provider-anthropic/thinking.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Anthropic extended-thinking configuration. Maps an optional - * `thinking_level` onto the Messages API `thinking` request field, with - * budgets from the model catalog (`thinking_budgets`) when present and a - * formula on the model's max output tokens as fallback. - * - * Invariant: Anthropic requires `max_tokens > thinking.budget_tokens` - * (the budget counts toward max_tokens) and a minimum budget of 1024. - * When the clamped request budget leaves no room, thinking is dropped - * instead of sending a request the API would reject. - */ - -import { logger } from '../runtime/otel.js'; -import type { Model, ThinkingBudgets } from '../models-catalog/types.js'; - -export type ThinkingConfig = { type: 'enabled'; budget_tokens: number }; - -/** Anthropic's documented minimum thinking budget. */ -export const MIN_THINKING_BUDGET = 1024; - -/** - * Tokens reserved for the visible answer. The thinking budget counts toward - * max_tokens; without a reserve an `xhigh` budget can consume the whole - * request budget and silently return an empty completion. - */ -export const OUTPUT_RESERVE_TOKENS = 1024; - -function budgetFromCatalog( - level: string, - budgets: ThinkingBudgets | undefined, -): number | undefined { - if (!budgets) return undefined; - switch (level) { - case 'minimal': - return budgets.minimal; - case 'low': - return budgets.low; - case 'medium': - return budgets.medium; - case 'high': - return budgets.high; - default: - return undefined; - } -} - -/** Budget formula from the model's max output tokens `output`. */ -function budgetFromFormula(level: string, output: number): number | undefined { - switch (level) { - case 'high': - return Math.min(16_000, Math.floor(output / 2 - 1)); - case 'max': - case 'xhigh': - return Math.min(31_999, output - 1); - case 'medium': - return Math.min(8_000, Math.floor(output / 4)); - case 'low': - case 'minimal': - return Math.min(4_000, Math.floor(output / 8)); - default: - return undefined; - } -} - -export function buildThinkingConfig( - level: string | undefined, - maxTokens: number, - catalog?: Model, -): ThinkingConfig | undefined { - if (!level || level === 'off') return undefined; - // An explicit `supports_thinking: false` would 400; unknown stays permissive. - if (catalog?.supports_thinking === false) return undefined; - // Degrade xhigh to the high tier when the catalog says xhigh is unsupported. - const effective = - (level === 'xhigh' || level === 'max') && catalog?.supports_xhigh === false ? 'high' : level; - - let budget = budgetFromCatalog(effective, catalog?.thinking_budgets); - if (budget === undefined || budget <= 0) { - const output = catalog?.max_output_tokens; - // The formula needs the model's output ceiling; unknown model -> no thinking. - if (typeof output !== 'number' || output <= 0) return undefined; - budget = budgetFromFormula(effective, output); - } - if (budget === undefined || budget <= 0) return undefined; - - // Keep budget below max_tokens with room for the visible answer; drop - // thinking (logged) when less than the API minimum remains. - budget = Math.min(budget, maxTokens - OUTPUT_RESERVE_TOKENS); - if (budget < MIN_THINKING_BUDGET) { - logger.debug('thinking dropped: budget below API minimum after max_tokens clamp', { - level, - maxTokens, - budget, - }); - return undefined; - } - - return { type: 'enabled', budget_tokens: budget }; -} diff --git a/harness/src/provider-anthropic/types.ts b/harness/src/provider-anthropic/types.ts deleted file mode 100644 index 9f626250f..000000000 --- a/harness/src/provider-anthropic/types.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Anthropic provider config + auth-mode union. Mirrors - * `provider-anthropic/src/lib.rs::{AnthropicConfig, AuthMode}` for the - * wire-relevant fields; `catalog` is a harness-side, in-process-only - * addition with no Rust counterpart (never serialized). - */ - -import type { Model } from '../models-catalog/types.js'; -import type { Credential } from '../runtime/provider-resolve.js'; -import { DEFAULT_API_URL } from './config.js'; - -export type AuthMode = 'api_key' | 'oauth_bearer'; - -export type AnthropicConfig = { - credential_value: string; - model: string; - max_tokens: number; - api_url: string; - auth_mode: AuthMode; - /** Catalog entry for `model` when known; used for thinking budgets. */ - catalog?: Model; -}; - -export function configWithCredential( - model: string, - cred: Credential, - max_tokens = 4096, - api_url = DEFAULT_API_URL, -): AnthropicConfig { - if (cred.type === 'api_key') { - return { credential_value: cred.key, model, max_tokens, api_url, auth_mode: 'api_key' }; - } - return { - credential_value: cred.access_token, - model, - max_tokens, - api_url, - auth_mode: 'oauth_bearer', - }; -} - -/** Pure helper: produces the `[headerName, headerValue]` pair for a config. */ -export function authHeaderFor(cfg: AnthropicConfig): [string, string] { - if (cfg.auth_mode === 'api_key') return ['x-api-key', cfg.credential_value]; - return ['authorization', `Bearer ${cfg.credential_value}`]; -} diff --git a/harness/src/provider-anthropic/wire-messages.ts b/harness/src/provider-anthropic/wire-messages.ts deleted file mode 100644 index ab4339420..000000000 --- a/harness/src/provider-anthropic/wire-messages.ts +++ /dev/null @@ -1,162 +0,0 @@ -/** - * AgentMessage → Anthropic wire shape. Mirrors - * `provider-anthropic/src/lib.rs::{to_wire_messages, content_block_to_wire, - * encode_tool_name, decode_tool_name}`. - */ - -import { logger } from '../runtime/otel.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import type { ContentBlock } from '../types/content.js'; -import { formatFunctionResultBlocks, formatFunctionResultContent } from '../types/wire.js'; - -/** - * Content shipped in the synthetic `tool_result` placeholder we inject - * when an assistant turn has a `tool_use` block with no matching - * `function_result` AgentMessage anywhere in the conversation. Anthropic - * rejects orphan tool_uses with: "tool_use IDs were found without - * tool_result blocks immediately after". This message lets the model - * understand the call was interrupted and move on. - */ -const ORPHAN_TOOL_PLACEHOLDER = - 'Tool call was interrupted before completing. Continue without its output.'; - -/** - * Anthropic's tool-name regex is `^[a-zA-Z0-9_-]{1,128}$`; bus ids use - * `::` separators. We replace `::` with `__` on the way out and reverse - * on the way back. (Tool names that already contain `__` are not in use - * today — see the Rust port for the same caveat.) - */ -export function encodeToolName(name: string): string { - return name.replaceAll('::', '__'); -} - -export function decodeToolName(name: string): string { - return name.replaceAll('__', '::'); -} - -export function contentBlockToWire(b: ContentBlock): unknown | null { - if (b.type === 'text') return { type: 'text', text: b.text }; - if (b.type === 'function_call') { - return { - type: 'tool_use', - id: b.id, - name: encodeToolName(b.function_id), - input: b.arguments, - }; - } - if (b.type === 'thinking') { - // During tool use Anthropic requires signed thinking blocks passed back - // unmodified (400 otherwise); unsigned blocks (aborted/partial stream) - // would fail signature verification and are dropped instead. - if (b.signature) return { type: 'thinking', thinking: b.text, signature: b.signature }; - return null; - } - return null; -} - -export function toWireMessages(messages: AgentMessage[]): unknown[] { - const out: unknown[] = []; - let pending: unknown[] = []; - const flush = () => { - if (pending.length > 0) { - out.push({ role: 'user', content: pending }); - pending = []; - } - }; - - // Boundary sanitization for orphan tool_uses. Pre-pass: collect every - // function_call_id that has a matching function_result AgentMessage. - // After emitting an assistant turn, any tool_use block whose id is NOT - // in this set gets a synthetic tool_result placeholder injected into - // the next user message — preventing Anthropic's - // "tool_use IDs were found without tool_result blocks immediately after" - // error. Investigation traced three orchestrator paths that can leave an - // orphan: abort during awaiting_approval, sync /compact mid-execution, - // and prepared-call desync. Defending at the wire layer covers all of - // them without coupling the fix to a specific state-machine edge. - const resolvedIds = new Set(); - for (const m of messages) { - if (m.role === 'function_result') resolvedIds.add(m.function_call_id); - } - - for (const m of messages) { - if (m.role === 'user') { - // Merge any pending tool_results (from prior function_result - // AgentMessages OR synthetic placeholders for orphan tool_uses) - // INTO this user message's content. Anthropic allows tool_result - // blocks and regular content in the same user message — and it - // forbids consecutive user messages, which a separate `flush()` - // would create. - const userContent = m.content.map(contentBlockToWire).filter((v): v is unknown => v !== null); - const content = [...pending, ...userContent]; - pending = []; - out.push({ role: 'user', content }); - } else if (m.role === 'assistant') { - flush(); - const content = m.content.map(contentBlockToWire).filter((v): v is unknown => v !== null); - out.push({ role: 'assistant', content }); - // Inject placeholders for any tool_use in this assistant that has - // no matching function_result. They land in `pending` and get - // flushed into the NEXT user message — exactly where Anthropic - // expects to find them. - for (const block of m.content) { - if (block.type !== 'function_call') continue; - if (resolvedIds.has(block.id)) continue; - logger.warn( - 'provider-anthropic: tool_use lacks matching function_result; injecting synthetic placeholder', - { tool_use_id: block.id, function_id: block.function_id }, - ); - pending.push({ - type: 'tool_result', - tool_use_id: block.id, - content: ORPHAN_TOOL_PLACEHOLDER, - is_error: true, - }); - // Mark resolved so a stray duplicate of the same orphan in a - // pathological message list doesn't get a second placeholder. - resolvedIds.add(block.id); - } - } else if (m.role === 'function_result') { - // Boundary dedup: even with idempotency guards upstream, never let a - // duplicate tool_result block reach Anthropic. The API rejects with - // "each tool_use must have a single result. Found multiple - // tool_result blocks with id: toolu_..." - // and the whole turn fails. Latest-wins: replace any existing block - // with the same tool_use_id in the current pending batch so the - // most recent function_result is what the model sees. - // Anthropic tool_result content accepts either a flat string or an - // array of text/image blocks. Keep the flat string whenever there - // are no images — that's the long-standing wire shape (and what - // prompt caching has seen) — and only switch to the array form when - // an image block must reach the model (e.g. web::fetch image mode). - const resultBlocks = formatFunctionResultBlocks(m); - const hasImages = resultBlocks.some((b) => b.type === 'image'); - const block = { - type: 'tool_result', - tool_use_id: m.function_call_id, - content: hasImages - ? resultBlocks.map((b) => - b.type === 'image' - ? { - type: 'image', - source: { type: 'base64', media_type: b.mime, data: b.data }, - } - : { type: 'text', text: b.text }, - ) - : formatFunctionResultContent(m), - is_error: m.is_error, - }; - const existingIdx = pending.findIndex( - (b) => (b as { tool_use_id?: string } | null)?.tool_use_id === m.function_call_id, - ); - if (existingIdx >= 0) { - pending[existingIdx] = block; - } else { - pending.push(block); - } - } - // custom messages are skipped - } - flush(); - return out; -} diff --git a/harness/src/provider-anthropic/wire-tools.ts b/harness/src/provider-anthropic/wire-tools.ts deleted file mode 100644 index 3394e2129..000000000 --- a/harness/src/provider-anthropic/wire-tools.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { AgentFunction } from '../types/function.js'; -import { encodeToolName } from './wire-messages.js'; - -export function functionsToWire(tools: AgentFunction[]): unknown[] { - return tools.map((t) => ({ - name: encodeToolName(t.name), - description: t.description, - input_schema: t.parameters, - })); -} diff --git a/harness/src/provider-kimi/auth.ts b/harness/src/provider-kimi/auth.ts deleted file mode 100644 index d40238a01..000000000 --- a/harness/src/provider-kimi/auth.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Resolve the Kimi (Moonshot) credential + runtime settings from the harness - * provider registry (`harness::provider::resolve`) and build a - * ChatCompletionsConfig. The provider id is `kimi`; the registry's env-var - * fallback maps it to MOONSHOT_API_KEY. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { clampOutputTokens, getCatalogModel } from '../runtime/output-tokens.js'; -import { resolveProvider } from '../runtime/provider-resolve.js'; -import type { WorkerConfig } from './config.js'; -import { type ChatCompletionsConfig, configFromCredential } from './types.js'; - -export const PROVIDER_ID = 'kimi'; - -export async function buildConfig( - iii: ISdk, - worker: WorkerConfig, - model: string, -): Promise { - const resolved = await resolveProvider(iii, PROVIDER_ID); - if (!resolved.credential) { - throw new Error( - 'harness::provider::resolve returned no credential for provider `kimi` ' + - '(set an api key in the harness configuration or MOONSHOT_API_KEY)', - ); - } - const apiUrl = resolved.api_url ?? worker.default_api_url; - const catalog = await getCatalogModel(iii, PROVIDER_ID, model); - const maxTokens = clampOutputTokens({ - modelMaxOutput: catalog?.max_output_tokens, - userOverride: resolved.max_tokens, - workerDefault: worker.default_max_tokens, - }); - return configFromCredential(apiUrl, PROVIDER_ID, model, resolved.credential, maxTokens); -} diff --git a/harness/src/provider-kimi/complete.ts b/harness/src/provider-kimi/complete.ts deleted file mode 100644 index 95d240a89..000000000 --- a/harness/src/provider-kimi/complete.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { requireString } from '../runtime/handler.js'; -import type { ISdk } from '../runtime/iii.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { collect, streamKimi } from './stream.js'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - 'provider::kimi::complete', - async (payload: unknown) => { - const obj = (payload ?? {}) as Record; - const model = requireString(obj, 'model'); - const system_prompt = typeof obj.system_prompt === 'string' ? obj.system_prompt : ''; - const messages = Array.isArray(obj.messages) ? (obj.messages as AgentMessage[]) : []; - const tools = Array.isArray(obj.tools) ? (obj.tools as AgentFunction[]) : []; - const cfg = await buildConfig(iii, worker, model); - return await collect(streamKimi({ cfg, system_prompt, messages, tools })); - }, - { - description: - 'Legacy: drain a streamed Kimi chat-completion and return the final AssistantMessage.', - }, - ); -} diff --git a/harness/src/provider-kimi/config.ts b/harness/src/provider-kimi/config.ts deleted file mode 100644 index 72e757f32..000000000 --- a/harness/src/provider-kimi/config.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { getNumber, getSection, getString } from '../runtime/config.js'; - -export type WorkerConfig = { - default_max_tokens: number; - default_api_url: string; -}; - -export const DEFAULT_API_URL = 'https://api.moonshot.ai/v1/chat/completions'; - -export function loadWorkerConfig(cfg: Record): WorkerConfig { - const section = getSection(cfg, 'provider_kimi'); - return { - default_max_tokens: getNumber(section, 'default_max_tokens', 8192), - default_api_url: getString(section, 'default_api_url', DEFAULT_API_URL), - }; -} diff --git a/harness/src/provider-kimi/discover.ts b/harness/src/provider-kimi/discover.ts deleted file mode 100644 index 633724143..000000000 --- a/harness/src/provider-kimi/discover.ts +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Kimi (Moonshot) model discovery — hits the OpenAI-compatible - * `GET /v1/models` and registers each returned model into the iii models - * catalog (cached for the picker) with a default context window. - * - * Best-effort: a missing credential or any upstream error yields `[]`. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { - deriveModelsUrl, - enrichModel, - fetchModelsForDiscovery, - type ModelStub, - reconcileModels, -} from '../runtime/models-discovery.js'; -import { getModelsDevIndex, lookupModelsDev } from '../runtime/modelsdev.js'; -import { logger } from '../runtime/otel.js'; -import { resolveProvider } from '../runtime/provider-resolve.js'; -import { PROVIDER_ID } from './auth.js'; -import type { WorkerConfig } from './config.js'; - -const DEFAULT_CONTEXT_WINDOW = 256_000; - -function parseStubs(json: unknown): ModelStub[] { - const data = (json as { data?: unknown })?.data; - if (!Array.isArray(data)) return []; - const out: ModelStub[] = []; - for (const raw of data as Array<{ id?: unknown }>) { - const id = typeof raw.id === 'string' && raw.id.length > 0 ? raw.id : null; - if (!id) continue; - out.push({ id }); - } - return out; -} - -export async function discoverAndRegister(iii: ISdk, worker: WorkerConfig): Promise { - const resolved = await resolveProvider(iii, PROVIDER_ID).catch(() => null); - const cred = resolved?.credential ?? null; - if (!cred) { - // No credential: drop any models a previous run registered so the picker - // reflects the removal instead of showing stale, unusable rows. - logger.info('kimi discovery: no credential; pruning catalog', {}); - await reconcileModels(iii, PROVIDER_ID, []); - return []; - } - const key = cred.type === 'api_key' ? cred.key : cred.access_token; - const url = deriveModelsUrl(resolved?.api_url ?? worker.default_api_url); - const fetchResult = await fetchModelsForDiscovery(url, { Authorization: `Bearer ${key}` }); - if (fetchResult.kind === 'auth_error') { - logger.info('kimi discovery: invalid credential; pruning catalog', { - status: fetchResult.status, - }); - await reconcileModels(iii, PROVIDER_ID, []); - return []; - } - if (fetchResult.kind !== 'ok') return []; - - const modelsDev = await getModelsDevIndex(); - const models = parseStubs(fetchResult.json).map((stub) => - enrichModel({ - provider: PROVIDER_ID, - api: 'openai-completions', - stub, - defaultContextWindow: DEFAULT_CONTEXT_WINDOW, - modelsDev: lookupModelsDev(modelsDev, PROVIDER_ID, stub.id), - }), - ); - const registered = await reconcileModels(iii, PROVIDER_ID, models); - logger.info('kimi discovery: reconciled models', { - count: registered.length, - discovered: models.length, - }); - return registered; -} diff --git a/harness/src/provider-kimi/iii.worker.yaml b/harness/src/provider-kimi/iii.worker.yaml deleted file mode 100644 index 160c24957..000000000 --- a/harness/src/provider-kimi/iii.worker.yaml +++ /dev/null @@ -1,17 +0,0 @@ -iii: v1 -name: provider-kimi -language: node -deploy: binary -manifest: package.json -bin: iii-provider-kimi -description: Kimi (Moonshot) Chat Completions streaming provider; exposes provider::kimi::stream and provider::kimi::complete on the iii bus. - -runtime: - kind: node - -scripts: - install: pnpm install - start: node ./dist/provider-kimi/main.js --config ./config.yaml - -dependencies: - configuration: "^0.11.0" diff --git a/harness/src/provider-kimi/main.ts b/harness/src/provider-kimi/main.ts deleted file mode 100644 index e8f90c1b8..000000000 --- a/harness/src/provider-kimi/main.ts +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env node -import { bootstrapWorker } from '../runtime/worker.js'; -import { register } from './register.js'; - -await bootstrapWorker({ - name: 'provider-kimi', - description: - 'Kimi (Moonshot) Chat Completions streaming provider on the iii bus (provider::kimi::stream + ::complete).', - register: (iii, ctx) => register(iii, ctx), -}); diff --git a/harness/src/provider-kimi/refresh-fn.ts b/harness/src/provider-kimi/refresh-fn.ts deleted file mode 100644 index 00567e20b..000000000 --- a/harness/src/provider-kimi/refresh-fn.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * `provider::kimi::refresh_models` — re-pull the upstream model list and - * register each into the iii models catalog. Returns `{ registered }`. - * Never throws across the bus boundary. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import type { WorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; - -export const FUNCTION_ID = 'provider::kimi::refresh_models'; - -export type RefreshResult = { registered: string[] }; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (): Promise => { - try { - return { registered: await discoverAndRegister(iii, worker) }; - } catch (err) { - logger.warn('provider::kimi::refresh_models failed', { err: String(err) }); - return { registered: [] }; - } - }, - { - description: - 'Re-pull the Kimi (Moonshot) model list (GET /v1/models) and register each into the iii models catalog. Idempotent.', - }, - ); -} diff --git a/harness/src/provider-kimi/register.ts b/harness/src/provider-kimi/register.ts deleted file mode 100644 index f155ce333..000000000 --- a/harness/src/provider-kimi/register.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { loadConfig } from '../runtime/config.js'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import { declareProvider } from '../runtime/provider-resolve.js'; -import { PROVIDER_ID } from './auth.js'; -import { register as registerComplete } from './complete.js'; -import { loadWorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; -import { register as registerRefresh } from './refresh-fn.js'; -import { register as registerStream } from './stream-fn.js'; - -export async function register(iii: ISdk, ctx: { configPath: string }): Promise { - const cfg = await loadConfig(ctx.configPath); - const worker = loadWorkerConfig(cfg); - registerComplete(iii, worker); - registerStream(iii, worker); - registerRefresh(iii, worker); - - void declareProvider(iii, { - id: PROVIDER_ID, - display_name: 'kimi (moonshot)', - credential_env_var: 'MOONSHOT_API_KEY', - defaults: { - api_url: worker.default_api_url, - max_tokens: worker.default_max_tokens, - }, - supports_model_listing: true, - }); - - setImmediate(() => { - discoverAndRegister(iii, worker).catch((err) => { - logger.warn('kimi startup discovery threw', { err: String(err) }); - }); - }); -} diff --git a/harness/src/provider-kimi/sse.ts b/harness/src/provider-kimi/sse.ts deleted file mode 100644 index 203a6ae12..000000000 --- a/harness/src/provider-kimi/sse.ts +++ /dev/null @@ -1,210 +0,0 @@ -// Kept separate from provider-openai so Moonshot-specific extensions -// (e.g. partial_mode) can land without coupling the two providers. - -import type { AssistantMessage } from '../types/agent-message.js'; -import type { ContentBlock } from '../types/content.js'; -import type { AssistantMessageEvent, ErrorKind, StopReason, Usage } from '../types/stream-event.js'; - -type PartialToolCall = { id: string; function_id: string; args_json: string }; - -export type PartialState = { - text: string; - /** - * Accumulated reasoning content emitted by Kimi K2 / K2.6 thinking mode - * via `delta.reasoning_content`. Persisted as a `thinking` ContentBlock - * on the AssistantMessage so subsequent turns can replay it back via - * the `reasoning_content` field on the wire — Kimi rejects assistant - * messages with tool_calls but no reasoning_content when thinking is - * enabled ("thinking is enabled but reasoning_content is missing in - * assistant tool call message at index N"). - */ - reasoning_text: string; - tool_calls: PartialToolCall[]; - usage: Usage; - stop_reason: StopReason; -}; - -export function emptyPartial(): PartialState { - return { - text: '', - reasoning_text: '', - tool_calls: [], - usage: { input: 0, output: 0, cache_read: 0, cache_write: 0 }, - stop_reason: 'end', - }; -} - -function buildContent(state: PartialState): ContentBlock[] { - const out: ContentBlock[] = []; - // Thinking goes FIRST so the provider-agnostic ContentBlock[] preserves - // the natural order: think → answer / tool_call. Kimi's wire format - // also lists reasoning_content before content / tool_calls. - if (state.reasoning_text.length > 0) { - out.push({ type: 'thinking', text: state.reasoning_text }); - } - if (state.text.length > 0) out.push({ type: 'text', text: state.text }); - for (const tc of state.tool_calls) { - if (tc.function_id.length === 0) continue; - let args: unknown = {}; - if (tc.args_json.length > 0) { - try { - args = JSON.parse(tc.args_json); - } catch { - args = null; - } - } - out.push({ type: 'function_call', id: tc.id, function_id: tc.function_id, arguments: args }); - } - return out; -} - -export function buildPartial( - state: PartialState, - model: string, - provider: string, -): AssistantMessage { - return { - role: 'assistant', - content: buildContent(state), - stop_reason: state.stop_reason, - error_message: null, - error_kind: null, - usage: state.usage, - model, - provider, - timestamp: Date.now(), - }; -} - -export function buildFinal(state: PartialState, model: string, provider: string): AssistantMessage { - return buildPartial(state, model, provider); -} - -export function mapFinishReason(s: string): StopReason { - if (s === 'stop') return 'end'; - if (s === 'length') return 'length'; - if (s === 'tool_calls' || s === 'function_call') return 'function_call'; - return 'end'; -} - -export function mergeUsage(usage: Record, into: Usage): void { - const num = (k: string) => (typeof usage[k] === 'number' ? (usage[k] as number) : 0); - into.input = (into.input ?? 0) + num('prompt_tokens') + num('input_tokens'); - into.output = (into.output ?? 0) + num('completion_tokens') + num('output_tokens'); - for (const parent of ['prompt_tokens_details', 'input_tokens_details']) { - const d = usage[parent] as Record | undefined; - if (d && typeof d.cached_tokens === 'number') { - into.cache_read = (into.cache_read ?? 0) + d.cached_tokens; - } - } -} - -export function classifyKimiError(message: string, status?: number): ErrorKind { - if (status === 401 || status === 403) return 'auth_expired'; - if (status === 429) return 'rate_limited'; - if (status && status >= 500) return 'transient'; - if (/context length|too many tokens/i.test(message)) return 'context_overflow'; - return 'permanent'; -} - -export function syntheticErrorEvent( - message: string, - model: string, - provider: string, - error_kind: ErrorKind = 'transient', -): AssistantMessageEvent { - const final: AssistantMessage = { - role: 'assistant', - content: [{ type: 'text', text: message }], - stop_reason: 'error', - error_message: message, - error_kind, - usage: null, - model, - provider, - timestamp: Date.now(), - }; - return { type: 'error', error: final }; -} - -export function handleChunk( - chunk: Record, - state: PartialState, - model: string, - provider: string, -): AssistantMessageEvent[] { - const events: AssistantMessageEvent[] = []; - const usage = chunk.usage as Record | undefined; - if (usage) mergeUsage(usage, state.usage); - const choices = chunk.choices; - if (!Array.isArray(choices) || choices.length === 0) return events; - const choice = choices[0] as Record; - const finish = typeof choice.finish_reason === 'string' ? choice.finish_reason : null; - if (finish) state.stop_reason = mapFinishReason(finish); - const delta = choice.delta as Record | undefined; - if (!delta) return events; - - // Reasoning tokens — Kimi K2.6 thinking mode streams these on - // `delta.reasoning_content` BEFORE any content/tool_calls. We surface - // them as thinking_* events (mirrors Anthropic's extended thinking - // shape) and persist them on the AssistantMessage so the next request - // can echo them back via `reasoning_content`, which Kimi requires on - // assistant tool-call messages when thinking is enabled. - if (typeof delta.reasoning_content === 'string' && delta.reasoning_content.length > 0) { - if (state.reasoning_text.length === 0) { - events.push({ type: 'thinking_start', partial: buildPartial(state, model, provider) }); - } - state.reasoning_text += delta.reasoning_content; - events.push({ - type: 'thinking_delta', - partial: buildPartial(state, model, provider), - delta: delta.reasoning_content, - }); - } - - if (typeof delta.content === 'string' && delta.content.length > 0) { - if (state.text.length === 0) { - events.push({ type: 'text_start', partial: buildPartial(state, model, provider) }); - } - state.text += delta.content; - events.push({ - type: 'text_delta', - partial: buildPartial(state, model, provider), - delta: delta.content, - }); - } - - const tool_calls = delta.tool_calls; - if (Array.isArray(tool_calls)) { - for (const tc of tool_calls) { - if (!tc || typeof tc !== 'object') continue; - const tcObj = tc as Record; - const rawIndex = typeof tcObj.index === 'number' ? tcObj.index : 0; - // Reject attacker-controlled indices that would force unbounded - // allocation. Same DoS guard as provider-lmstudio/sse.ts. - if (!Number.isInteger(rawIndex) || rawIndex < 0 || rawIndex > 256) { - continue; - } - const index = rawIndex; - while (state.tool_calls.length <= index) { - state.tool_calls.push({ id: '', function_id: '', args_json: '' }); - } - const entry = state.tool_calls[index]; - if (!entry) continue; - if (typeof tcObj.id === 'string' && tcObj.id.length > 0) entry.id = tcObj.id; - const fn = tcObj.function as Record | undefined; - if (fn) { - if (typeof fn.name === 'string' && fn.name.length > 0) entry.function_id = fn.name; - if (typeof fn.arguments === 'string') { - entry.args_json += fn.arguments; - events.push({ - type: 'functioncall_delta', - partial: buildPartial(state, model, provider), - delta: fn.arguments, - }); - } - } - } - } - return events; -} diff --git a/harness/src/provider-kimi/stream-fn.ts b/harness/src/provider-kimi/stream-fn.ts deleted file mode 100644 index e8aa467d2..000000000 --- a/harness/src/provider-kimi/stream-fn.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { ChannelWriter } from 'iii-sdk'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import { - ProviderStreamInputJsonSchema, - ProviderStreamOutputJsonSchema, - ProviderStreamRuntimeInputSchema, -} from '../types/provider.js'; -import { isTerminal } from '../types/stream-event.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { streamKimi } from './stream.js'; - -export const FUNCTION_ID = 'provider::kimi::stream'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (raw: unknown) => { - const input = ProviderStreamRuntimeInputSchema.parse(raw); - // The iii-sdk auto-hydrates `writer_ref` (a StreamChannelRef on the - // wire) into a `ChannelWriter` instance before this handler runs. - const writer = input.writer_ref as ChannelWriter; - const cfg = await buildConfig(iii, worker, input.model); - try { - const events = streamKimi({ - cfg, - system_prompt: input.system_prompt ?? '', - messages: input.messages as AgentMessage[], - tools: input.tools as import('../types/function.js').AgentFunction[], - }); - for await (const ev of events) { - writer.sendMessage(JSON.stringify(ev)); - if (isTerminal(ev)) break; - } - } catch (err) { - logger.warn('provider::kimi::stream failed mid-flight', { err: String(err) }); - } finally { - try { - writer.close(); - } catch (err) { - logger.debug('writer.close failed', { err: String(err) }); - } - } - return { ok: true }; - }, - { - description: - 'Stream a single assistant turn from Kimi (Moonshot) Chat Completions into the caller-supplied channel.', - request_format: ProviderStreamInputJsonSchema as Record, - response_format: ProviderStreamOutputJsonSchema as Record, - }, - ); -} diff --git a/harness/src/provider-kimi/stream.ts b/harness/src/provider-kimi/stream.ts deleted file mode 100644 index e85700eb1..000000000 --- a/harness/src/provider-kimi/stream.ts +++ /dev/null @@ -1,156 +0,0 @@ -// Kept separate from provider-openai so Moonshot-specific request options -// can be added without coupling the two providers. - -import { logger } from '../runtime/otel.js'; -import type { AgentMessage, AssistantMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import type { AssistantMessageEvent } from '../types/stream-event.js'; -import { - buildFinal, - classifyKimiError, - emptyPartial, - handleChunk, - syntheticErrorEvent, -} from './sse.js'; -import type { ChatCompletionsConfig } from './types.js'; -import { toOpenaiMessages } from './wire-messages.js'; -import { functionsToOpenai } from './wire-tools.js'; - -export type StreamArgs = { - cfg: ChatCompletionsConfig; - system_prompt: string; - messages: AgentMessage[]; - tools: AgentFunction[]; -}; - -export async function* streamKimi({ - cfg, - system_prompt, - messages, - tools, -}: StreamArgs): AsyncGenerator { - const body: Record = { - model: cfg.model, - max_completion_tokens: cfg.max_tokens, - messages: toOpenaiMessages(messages, system_prompt), - stream: true, - stream_options: { include_usage: true }, - }; - if (tools.length > 0) body.tools = functionsToOpenai(tools); - - const authName = cfg.auth_header_name ?? 'Authorization'; - const authPrefix = cfg.auth_value_prefix ?? 'Bearer '; - const headers: Record = { - 'content-type': 'application/json', - [authName]: `${authPrefix}${cfg.api_key}`, - }; - for (const [k, v] of cfg.extra_headers ?? []) headers[k] = v; - - let resp: Response; - try { - resp = await fetch(cfg.url, { - method: 'POST', - headers, - body: JSON.stringify(body), - }); - } catch (err) { - yield syntheticErrorEvent(`kimi fetch failed: ${String(err)}`, cfg.model, cfg.provider_name); - return; - } - if (!resp.ok) { - const text = await resp.text().catch(() => ''); - yield syntheticErrorEvent( - text || `kimi http ${resp.status}`, - cfg.model, - cfg.provider_name, - classifyKimiError(text, resp.status), - ); - return; - } - const partial: AssistantMessage = { - role: 'assistant', - content: [], - stop_reason: 'end', - error_message: null, - error_kind: null, - usage: null, - model: cfg.model, - provider: cfg.provider_name, - timestamp: Date.now(), - }; - yield { type: 'start', partial }; - - const state = emptyPartial(); - if (!resp.body) { - yield syntheticErrorEvent('kimi response missing body', cfg.model, cfg.provider_name); - return; - } - const reader = resp.body.getReader(); - const decoder = new TextDecoder(); - let buf = ''; - try { - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - let idx = buf.indexOf('\n\n'); - while (idx >= 0) { - const block = buf.slice(0, idx); - buf = buf.slice(idx + 2); - const dataLine = parseDataLine(block); - idx = buf.indexOf('\n\n'); - if (dataLine === null) continue; - if (dataLine === '[DONE]') { - yield { type: 'done', message: buildFinal(state, cfg.model, cfg.provider_name) }; - return; - } - let parsed: Record | null = null; - try { - parsed = JSON.parse(dataLine) as Record; - } catch { - continue; - } - if (parsed) { - for (const e of handleChunk(parsed, state, cfg.model, cfg.provider_name)) yield e; - } - } - } - } catch (err) { - logger.warn('kimi stream read failed', { err: String(err) }); - yield syntheticErrorEvent(`stream read failed: ${String(err)}`, cfg.model, cfg.provider_name); - return; - } - yield { type: 'done', message: buildFinal(state, cfg.model, cfg.provider_name) }; -} - -function parseDataLine(block: string): string | null { - let data: string | null = null; - for (const line of block.split('\n')) { - if (line.startsWith('data: ')) data = line.slice('data: '.length); - } - return data; -} - -export async function collect( - events: AsyncIterable, -): Promise { - let last: AssistantMessage | null = null; - for await (const ev of events) { - if (ev.type === 'done') return ev.message; - if (ev.type === 'error') return ev.error; - if ('partial' in ev) last = ev.partial; - } - return ( - last ?? { - role: 'assistant', - content: [], - stop_reason: 'error', - error_message: 'stream closed without final', - error_kind: 'transient', - usage: null, - model: 'kimi', - provider: 'kimi', - timestamp: Date.now(), - } - ); -} diff --git a/harness/src/provider-kimi/types.ts b/harness/src/provider-kimi/types.ts deleted file mode 100644 index 6884b3531..000000000 --- a/harness/src/provider-kimi/types.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { Credential } from '../runtime/provider-resolve.js'; - -export type ChatCompletionsConfig = { - url: string; - provider_name: string; - model: string; - api_key: string; - /** Defaults to "Authorization". */ - auth_header_name?: string; - /** Defaults to "Bearer ". */ - auth_value_prefix?: string; - extra_headers?: Array; - max_tokens: number; -}; - -export function configFromCredential( - url: string, - provider_name: string, - model: string, - cred: Credential, - max_tokens: number, -): ChatCompletionsConfig { - const api_key = cred.type === 'api_key' ? cred.key : cred.access_token; - return { url, provider_name, model, api_key, max_tokens }; -} diff --git a/harness/src/provider-kimi/wire-messages.ts b/harness/src/provider-kimi/wire-messages.ts deleted file mode 100644 index c6b6f0c14..000000000 --- a/harness/src/provider-kimi/wire-messages.ts +++ /dev/null @@ -1,79 +0,0 @@ -// Kept separate from provider-openai so Moonshot-specific extensions -// can land without coupling the two providers. - -import type { AgentMessage } from '../types/agent-message.js'; -import { formatFunctionResultContent } from '../types/wire.js'; - -export function toOpenaiMessages(messages: AgentMessage[], system_prompt: string): unknown[] { - const out: unknown[] = []; - if (system_prompt.length > 0) { - out.push({ role: 'system', content: system_prompt }); - } - for (const m of messages) { - if (m.role === 'user') { - const text = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'text' }> => c.type === 'text', - ) - .map((c) => c.text) - .join('\n'); - out.push({ role: 'user', content: text }); - } else if (m.role === 'assistant') { - const text = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'text' }> => c.type === 'text', - ) - .map((c) => c.text) - .join('\n'); - // Kimi K2 thinking mode: when this assistant turn produced reasoning, - // we MUST echo it back on subsequent requests under `reasoning_content` - // — otherwise Kimi rejects with "thinking is enabled but - // reasoning_content is missing in assistant tool call message". - // sse.ts persists it as a ThinkingContent block; we project that here. - const reasoning = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'thinking' }> => - c.type === 'thinking', - ) - .map((c) => c.text) - .join(''); - const tool_calls = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'function_call' }> => - c.type === 'function_call', - ) - .map((c) => ({ - id: c.id, - type: 'function', - function: { name: c.function_id, arguments: JSON.stringify(c.arguments) }, - })); - const entry: Record = { role: 'assistant' }; - if (reasoning.length > 0) entry.reasoning_content = reasoning; - if (text.length > 0) entry.content = text; - if (tool_calls.length > 0) entry.tool_calls = tool_calls; - out.push(entry); - } else if (m.role === 'function_result') { - const text = formatFunctionResultContent(m); - const row: Record = { - role: 'tool', - tool_call_id: m.function_call_id, - content: text, - }; - if (m.is_error) row.is_error = true; - // Boundary dedup — see provider-anthropic/wire-messages.ts for why. - // Latest-wins replace, so the freshest tool result is what Kimi sees. - const existingIdx = out.findIndex( - (e) => - (e as { role?: string }).role === 'tool' && - (e as { tool_call_id?: string }).tool_call_id === m.function_call_id, - ); - if (existingIdx >= 0) { - out[existingIdx] = row; - } else { - out.push(row); - } - } - // custom messages are skipped - } - return out; -} diff --git a/harness/src/provider-kimi/wire-tools.ts b/harness/src/provider-kimi/wire-tools.ts deleted file mode 100644 index c133e4f33..000000000 --- a/harness/src/provider-kimi/wire-tools.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { AgentFunction } from '../types/function.js'; - -export function functionsToOpenai(functions: AgentFunction[]): unknown[] { - return functions.map((t) => ({ - type: 'function', - function: { - name: t.name, - description: t.description, - parameters: t.parameters, - }, - })); -} diff --git a/harness/src/provider-llamacpp/auth.ts b/harness/src/provider-llamacpp/auth.ts deleted file mode 100644 index 90fc52bde..000000000 --- a/harness/src/provider-llamacpp/auth.ts +++ /dev/null @@ -1,155 +0,0 @@ -import type { ISdk } from '../runtime/iii.js'; -import { normalizeChatCompletionsUrl } from '../runtime/openai-compat-url.js'; -import { logger } from '../runtime/otel.js'; -import { clampOutputTokens, getCatalogModel } from '../runtime/output-tokens.js'; -import { - type Credential, - type ProviderResolveResult, - resolveProvider, -} from '../runtime/provider-resolve.js'; -import type { WorkerConfig } from './config.js'; -import { type ChatCompletionsConfig, configFromCredential } from './types.js'; - -// llama-server is local-first by default. Unlike LM Studio, llama.cpp has -// no documented "default" bearer string — the server either enforces a -// shared `--api-key` (in which case the caller must match it) or accepts -// any/no token. We therefore: -// - use the configured api key / LLAMACPP_API_KEY if present (set when the -// user started llama-server with `--api-key …`) -// - on loopback without a key, omit Authorization entirely -// - on non-loopback without a key, omit AND warn (don't ship a -// synthetic bearer to an arbitrary host) -export const PROVIDER_ID = 'llamacpp'; - -const EMPTY_RESOLVE: ProviderResolveResult = { - configured: false, - source: null, - credential: null, - api_url: null, - max_tokens: null, -}; - -function extractKey(cred: Credential | null): string { - if (!cred) return ''; - return cred.type === 'api_key' ? cred.key : cred.access_token; -} - -/** - * `true` when `url` resolves to a localhost / loopback target. Used to - * decide whether to warn when no credential is configured. - */ -export function isLoopbackUrl(url: string): boolean { - let parsed: URL; - try { - parsed = new URL(url); - } catch { - return false; - } - const host = parsed.hostname.toLowerCase(); - if (host === 'localhost') return true; - if (host === '127.0.0.1' || host === '::1' || host === '[::1]') return true; - if (host.endsWith('.localhost')) return true; - if (/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host)) return true; - return false; -} - -/** - * Resolve this provider's credential + settings via the harness registry. - * Tolerant: llama-server commonly runs unauthenticated on loopback, so a - * missing harness/registry yields an empty result rather than throwing. - */ -async function resolveTolerant(iii: ISdk): Promise { - try { - return await resolveProvider(iii, PROVIDER_ID); - } catch (err) { - logger.warn('llamacpp.auth: resolve failed; falling back to no-credential', { - code: 'llamacpp_auth_fetch_failed', - err: String(err), - }); - return EMPTY_RESOLVE; - } -} - -export async function fetchCredential(iii: ISdk): Promise { - return (await resolveTolerant(iii)).credential; -} - -/** - * Decide which API key (if any) to send for `url`. - * - * - Explicit credential present → use it. - * - Otherwise loopback → null (omit Authorization). llama-server without - * `--api-key` accepts unauthenticated requests; we don't manufacture - * a synthetic token. - * - Otherwise (non-loopback, no explicit credential) → null AND a - * warning log so operators see they're hitting a remote without - * credentials. - */ -export function selectAuthKey(cred: Credential | null, url: string): string | null { - const key = extractKey(cred); - if (key.length > 0) return key; - if (isLoopbackUrl(url)) return null; - logger.warn( - 'llamacpp.auth: no credential configured AND LLAMACPP_BASE_URL is non-loopback; omitting Authorization header', - { - code: 'llamacpp_auth_omitted_nonloopback', - origin: (() => { - try { - const u = new URL(url); - return `${u.protocol}//${u.host}`; - } catch { - return ''; - } - })(), - }, - ); - return null; -} - -export async function buildConfig( - iii: ISdk, - worker: WorkerConfig, - model: string, -): Promise { - const resolved = await resolveTolerant(iii); - const cred = resolved.credential; - // Normalise the override URL so a base-URL save from the config UI - // (e.g. `http://host:8080`) gets `/v1/chat/completions` appended. - // worker.default_api_url is already normalised in resolveApiUrl. - const overrideUrl = resolved.api_url ? normalizeChatCompletionsUrl(resolved.api_url) : null; - const apiUrl = overrideUrl ?? worker.default_api_url; - const catalog = await getCatalogModel(iii, PROVIDER_ID, model); - const maxTokens = clampOutputTokens({ - modelMaxOutput: catalog?.max_output_tokens, - userOverride: resolved.max_tokens, - workerDefault: worker.default_max_tokens, - }); - // Pass the credential through verbatim so configFromCredential can - // populate api_key (empty string when no credential). Header emission - // is gated separately in buildAuthHeaders/the stream layer. - return configFromCredential( - apiUrl, - 'llamacpp', - model, - extractKey(cred).length > 0 ? cred : null, - maxTokens, - ); -} - -/** - * Build the HTTP headers used for any llama-server REST call (chat - * completions AND `/v1/models` discovery). Shared so the auth dance — - * credential lookup, loopback-vs-remote decision — lives in exactly - * one place. Authorization is omitted when no credential applies. - */ -export async function buildAuthHeaders(iii: ISdk, url: string): Promise> { - const cred = await fetchCredential(iii); - const token = selectAuthKey(cred, url); - const base: Record = { - 'content-type': 'application/json', - }; - if (token !== null) { - base.Authorization = `Bearer ${token}`; - } - return base; -} diff --git a/harness/src/provider-llamacpp/complete.ts b/harness/src/provider-llamacpp/complete.ts deleted file mode 100644 index 8ffea2f23..000000000 --- a/harness/src/provider-llamacpp/complete.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { requireString } from '../runtime/handler.js'; -import type { ISdk } from '../runtime/iii.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { collect, streamLlamacpp } from './stream.js'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - 'provider::llamacpp::complete', - async (payload: unknown) => { - const obj = (payload ?? {}) as Record; - const model = requireString(obj, 'model'); - const system_prompt = typeof obj.system_prompt === 'string' ? obj.system_prompt : ''; - const messages = Array.isArray(obj.messages) ? (obj.messages as AgentMessage[]) : []; - const tools = Array.isArray(obj.tools) ? (obj.tools as AgentFunction[]) : []; - const cfg = await buildConfig(iii, worker, model); - return await collect(streamLlamacpp({ cfg, system_prompt, messages, tools })); - }, - { - description: - 'Legacy: drain a streamed llama-server chat-completion and return the final AssistantMessage.', - }, - ); -} diff --git a/harness/src/provider-llamacpp/config.ts b/harness/src/provider-llamacpp/config.ts deleted file mode 100644 index 37b5a91d0..000000000 --- a/harness/src/provider-llamacpp/config.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { logger } from '../runtime/otel.js'; -import { getNumber, getSection, getString } from '../runtime/config.js'; - -export type WorkerConfig = { - default_max_tokens: number; - default_api_url: string; -}; - -// llama-server's default port is 8080. Anyone running multiple inference -// servers will likely override this — see config.yaml / LLAMACPP_BASE_URL. -export const DEFAULT_API_URL = 'http://localhost:8080/v1/chat/completions'; - -/** - * Validate a candidate URL: parses cleanly AND uses an http/https - * scheme. Returns the parsed URL if valid, null otherwise. - * - * Same security posture as provider-lmstudio: a malformed or - * attacker-controlled `LLAMACPP_BASE_URL` (stale EnvironmentFile, typo - * with extra leading scheme) is rejected so we never concatenate an - * unvalidated string into the request target. - */ -function validatedUrl(raw: string): URL | null { - let parsed: URL; - try { - parsed = new URL(raw); - } catch { - return null; - } - if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { - return null; - } - return parsed; -} - -function isLoopbackHost(host: string): boolean { - const h = host.toLowerCase(); - if (h === 'localhost' || h.endsWith('.localhost')) return true; - if (h === '127.0.0.1' || h === '::1' || h === '[::1]') return true; - if (/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(h)) return true; - return false; -} - -/** - * Resolve the llama-server base URL with this precedence: - * 1. `LLAMACPP_BASE_URL` env var (per-machine override — wins over yaml) - * 2. `provider_llamacpp.default_api_url` in config.yaml - * 3. The localhost DEFAULT_API_URL constant above - * - * The env var accepts either a base origin (`http://host:port`) or a - * full URL (`http://host:port/v1/chat/completions`). When only a base - * is given, `/v1/chat/completions` is appended automatically. - * - * Both the env value and the yaml value are validated as http(s) URLs; - * a malformed value falls back to the next tier rather than being - * concatenated raw. A warning is logged when the resolved host is not - * loopback so operators see they're shipping their bearer to a remote. - */ -function resolveApiUrl(yamlValue: string): string { - const envRaw = (process.env.LLAMACPP_BASE_URL ?? '').trim(); - let candidate: string | null = null; - if (envRaw.length > 0) { - const trimmedEnv = envRaw.endsWith('/') ? envRaw.slice(0, -1) : envRaw; - const withPath = trimmedEnv.includes('/chat/completions') - ? trimmedEnv - : `${trimmedEnv}/v1/chat/completions`; - if (validatedUrl(withPath)) { - candidate = withPath; - } else { - logger.warn('llamacpp.config: LLAMACPP_BASE_URL is not a valid http(s) URL — ignoring', { - code: 'llamacpp_base_url_invalid', - }); - } - } - if (candidate === null) candidate = yamlValue; - const parsed = validatedUrl(candidate); - if (!parsed) { - logger.warn( - 'llamacpp.config: resolved API URL is not a valid http(s) URL; falling back to default', - { code: 'llamacpp_api_url_invalid' }, - ); - return DEFAULT_API_URL; - } - if (!isLoopbackHost(parsed.hostname)) { - logger.warn( - 'llamacpp.config: API URL is non-loopback — bearer (if configured) will be sent to a remote host', - { - code: 'llamacpp_api_url_remote', - origin: `${parsed.protocol}//${parsed.host}`, - }, - ); - } - return candidate; -} - -export function loadWorkerConfig(cfg: Record): WorkerConfig { - const section = getSection(cfg, 'provider_llamacpp'); - const yamlUrl = getString(section, 'default_api_url', DEFAULT_API_URL); - return { - default_max_tokens: getNumber(section, 'default_max_tokens', 8192), - default_api_url: resolveApiUrl(yamlUrl), - }; -} diff --git a/harness/src/provider-llamacpp/discover.ts b/harness/src/provider-llamacpp/discover.ts deleted file mode 100644 index 782d4fc24..000000000 --- a/harness/src/provider-llamacpp/discover.ts +++ /dev/null @@ -1,279 +0,0 @@ -/** - * llama-server model discovery — hits OpenAI-compatible `GET /v1/models` - * and registers the (single) currently-loaded model into the iii models - * catalog so the picker shows the real id (e.g. `Meta-Llama-3.1-8B`) - * instead of just the `llamacpp-local` placeholder. - * - * Unlike provider-lmstudio: - * - llama-server runs exactly ONE model at process startup (set via - * `-m model.gguf`), so the endpoint returns at most one entry. There - * is no concept of "downloaded but not loaded". - * - There is no v0-style endpoint that surfaces per-model context - * length in `/v1/models`, but the server-wide `GET /props` endpoint - * does expose `n_ctx` (the value passed via `-c` / `--ctx-size` at - * startup), shared across all slots. We fetch it alongside the - * models list and use it to populate `context_window`; otherwise we - * fall back to the embedded catalog placeholder (`llamacpp-local`). - * - * Best-effort: failures (server offline, malformed JSON, register RPC - * errors) are logged and swallowed — the worker still boots, and the - * embedded placeholder remains usable as a fallback. - */ - -import type { Model } from '../models-catalog/types.js'; -import type { ISdk } from '../runtime/iii.js'; -import { reconcileModels } from '../runtime/models-discovery.js'; -import { logger } from '../runtime/otel.js'; -import { PROVIDER_ID } from './auth.js'; - -const DISCOVERY_TIMEOUT_MS = 5_000; -const DEFAULT_CONTEXT_WINDOW = 32_768; -const DEFAULT_MAX_OUTPUT_TOKENS = 8_192; - -/** OpenAI-compatible `/v1/models` per-entry shape, parsed defensively. */ -type OpenAiModel = { - id?: unknown; - object?: unknown; -}; - -type OpenAiModelsResponse = { - data?: unknown; -}; - -async function fetchWithTimeout( - url: string, - init: RequestInit, - timeoutMs: number, -): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), timeoutMs); - try { - return await fetch(url, { ...init, signal: controller.signal }); - } finally { - clearTimeout(timer); - } -} - -/** - * Derive the sibling `/v1/models` endpoint from a chat-completions URL - * by trimming the trailing `/chat/completions`. Works for the default - * `…/v1/chat/completions` and for any custom-path proxy that follows - * the same convention. - */ -export function modelsUrl(chatUrl: string): string { - const trimmed = chatUrl.replace(/\/chat\/completions\/?$/, ''); - return `${trimmed}/models`; -} - -/** - * Derive the server-wide `/props` endpoint from a chat-completions URL. - * - * `/props` lives at the server root (not under `/v1`), so we strip both - * a `…/chat/completions` suffix and a trailing `/v1` (or any other path - * segment), then append `/props`. - * - * Examples: - * http://host:8080/v1/chat/completions → http://host:8080/props - * http://host:8080/v1/ → http://host:8080/props - * http://host:8080/custom/path → http://host:8080/custom/path/props - */ -export function propsUrl(chatUrl: string): string { - const withoutChat = chatUrl.replace(/\/chat\/completions\/?$/, ''); - const withoutV1 = withoutChat.replace(/\/v1\/?$/, ''); - const trimmed = withoutV1.endsWith('/') ? withoutV1.slice(0, -1) : withoutV1; - return `${trimmed}/props`; -} - -/** - * Fetch and parse the loaded model list. Returns an empty array on any - * failure (caller decides whether to surface it). - */ -async function fetchModelIds( - modelsEndpoint: string, - headers: Record, -): Promise { - let resp: Response; - try { - resp = await fetchWithTimeout(modelsEndpoint, { method: 'GET', headers }, DISCOVERY_TIMEOUT_MS); - } catch (err) { - logger.warn('llamacpp discovery: fetch failed', { - url: modelsEndpoint, - err: String(err), - }); - return []; - } - if (!resp.ok) { - logger.warn('llamacpp discovery: non-2xx response', { - url: modelsEndpoint, - status: resp.status, - }); - return []; - } - let parsed: OpenAiModelsResponse; - try { - parsed = (await resp.json()) as OpenAiModelsResponse; - } catch (err) { - logger.warn('llamacpp discovery: malformed JSON response', { - url: modelsEndpoint, - err: String(err), - }); - return []; - } - if (!Array.isArray(parsed.data)) return []; - const ids: string[] = []; - for (const entry of parsed.data) { - if (!entry || typeof entry !== 'object') continue; - const e = entry as OpenAiModel; - if (typeof e.id === 'string' && e.id.length > 0) { - ids.push(e.id); - } - } - return ids; -} - -/** - * `GET /props` response shape (only the fields we read). `n_ctx` is the - * server-wide context size from `-c` / `--ctx-size` at startup and is - * shared across slots — applies to whichever single model llama-server - * has loaded. Older llama-server builds only expose `n_ctx` nested in - * `default_generation_settings`; newer builds also expose it at the top - * level. We read both and prefer the top-level field when present. - */ -type LlamaProps = { - n_ctx?: unknown; - default_generation_settings?: unknown; -}; - -function isPositiveInteger(value: unknown): value is number { - return typeof value === 'number' && Number.isFinite(value) && value > 0; -} - -function readNestedNCtx(value: unknown): number | null { - if (!value || typeof value !== 'object') return null; - const v = (value as Record).n_ctx; - return isPositiveInteger(v) ? v : null; -} - -/** - * Fetch the server-wide context window from `GET /props`. Returns the - * positive integer `n_ctx`, or `null` on any failure (server offline, - * non-2xx, malformed JSON, missing/non-positive field). Best-effort — - * a `null` return causes the caller to fall back to DEFAULT_CONTEXT_WINDOW. - */ -async function fetchPropsContextWindow( - endpoint: string, - headers: Record, -): Promise { - let resp: Response; - try { - resp = await fetchWithTimeout(endpoint, { method: 'GET', headers }, DISCOVERY_TIMEOUT_MS); - } catch (err) { - logger.warn('llamacpp discovery: /props fetch failed', { - url: endpoint, - err: String(err), - }); - return null; - } - if (!resp.ok) { - logger.warn('llamacpp discovery: /props non-2xx response', { - url: endpoint, - status: resp.status, - }); - return null; - } - let parsed: LlamaProps; - try { - parsed = (await resp.json()) as LlamaProps; - } catch (err) { - logger.warn('llamacpp discovery: /props malformed JSON response', { - url: endpoint, - err: String(err), - }); - return null; - } - if (!parsed || typeof parsed !== 'object') return null; - if (isPositiveInteger(parsed.n_ctx)) return parsed.n_ctx; - const nested = readNestedNCtx(parsed.default_generation_settings); - if (nested !== null) return nested; - logger.warn('llamacpp discovery: /props response missing usable n_ctx', { - url: endpoint, - }); - return null; -} - -/** - * Build a placeholder-shaped Model row for each id returned by - * /v1/models. Field defaults match the `llamacpp-local` catalog - * placeholder so capability gating (supports_tools etc.) works for - * arbitrary user-loaded models. `contextWindow` is the value from - * `/props.n_ctx` when available; callers pass DEFAULT_CONTEXT_WINDOW - * as a fallback. - */ -function toCatalogModel(id: string, contextWindow: number): Model { - return { - id, - provider: 'llamacpp', - api: 'openai-completions', - display_name: id, - context_window: contextWindow, - max_output_tokens: DEFAULT_MAX_OUTPUT_TOKENS, - supports_thinking: false, - supports_xhigh: false, - supports_tools: true, - supports_vision: false, - supports_cache: false, - transports: ['sse'], - }; -} - -export async function discoverLoadedModel( - chatUrl: string, - headers: Record, -): Promise { - const [ids, ctx] = await Promise.all([ - fetchModelIds(modelsUrl(chatUrl), headers), - fetchPropsContextWindow(propsUrl(chatUrl), headers), - ]); - const contextWindow = ctx ?? DEFAULT_CONTEXT_WINDOW; - // Visible at INFO so operators can tell from the harness log whether - // /props reported the real n_ctx or we fell through to the default — - // distinguishes "served context too small" from "discovery couldn't - // see n_ctx" without needing to attach a debugger. - logger.info('llamacpp discovery: resolved context window', { - context_window: contextWindow, - source: ctx === null ? 'default_fallback' : 'props_n_ctx', - }); - return ids.map((id) => toCatalogModel(id, contextWindow)); -} - -/** Register discovered models in one `models::reconcile` call. */ -export async function registerDiscovered(iii: ISdk, models: readonly Model[]): Promise { - if (models.length === 0) return []; - const provider = models[0]?.provider; - if (!provider) return []; - return reconcileModels(iii, provider, models); -} - -/** - * One-shot: discover the loaded llama-server model and register it - * into the iii models catalog. Used at worker startup (fire-and-forget) - * and by `provider::llamacpp::refresh_models` on demand. - */ -export async function discoverAndRegister( - iii: ISdk, - chatUrl: string, - headers: Record, -): Promise { - const models = await discoverLoadedModel(chatUrl, headers); - if (models.length === 0) { - // Empty can mean "server offline" or "no model loaded" — we can't tell - // them apart here, so keep the last-known catalog rather than risk wiping - // it on a transient blip. - logger.info('llamacpp discovery: no loaded model found', {}); - return []; - } - const registered = await reconcileModels(iii, PROVIDER_ID, models); - logger.info('llamacpp discovery: registered models', { count: registered.length }); - logger.debug('llamacpp discovery: registered model ids', { ids: registered }); - return registered; -} diff --git a/harness/src/provider-llamacpp/iii.worker.yaml b/harness/src/provider-llamacpp/iii.worker.yaml deleted file mode 100644 index 38406c807..000000000 --- a/harness/src/provider-llamacpp/iii.worker.yaml +++ /dev/null @@ -1,17 +0,0 @@ -iii: v1 -name: provider-llamacpp -language: node -deploy: binary -manifest: package.json -bin: iii-provider-llamacpp -description: llama.cpp llama-server (localhost) Chat Completions streaming provider; exposes provider::llamacpp::stream and provider::llamacpp::complete on the iii bus. - -runtime: - kind: node - -scripts: - install: pnpm install - start: node ./dist/provider-llamacpp/main.js --config ./config.yaml - -dependencies: - configuration: "^0.11.0" diff --git a/harness/src/provider-llamacpp/main.ts b/harness/src/provider-llamacpp/main.ts deleted file mode 100644 index 5329c9569..000000000 --- a/harness/src/provider-llamacpp/main.ts +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env node -import { bootstrapWorker } from '../runtime/worker.js'; -import { register } from './register.js'; - -await bootstrapWorker({ - name: 'provider-llamacpp', - description: - 'llama.cpp llama-server (localhost) Chat Completions streaming provider on the iii bus (provider::llamacpp::stream + ::complete).', - register: (iii, ctx) => register(iii, ctx), -}); diff --git a/harness/src/provider-llamacpp/refresh-fn.ts b/harness/src/provider-llamacpp/refresh-fn.ts deleted file mode 100644 index c1697adfb..000000000 --- a/harness/src/provider-llamacpp/refresh-fn.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * `provider::llamacpp::refresh_models` — bus function the UI (or a - * script) can call to re-discover the loaded llama-server model - * without restarting this worker. Wraps `discoverAndRegister`. - * - * Returns `{ registered: string[] }` — the IDs (typically one, since - * llama-server hosts a single model per process) of all models that - * were (re-)written into the catalog on this call. Idempotent. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import { buildAuthHeaders } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; - -export const FUNCTION_ID = 'provider::llamacpp::refresh_models'; - -export type RefreshResult = { - registered: string[]; -}; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (): Promise => { - try { - const headers = await buildAuthHeaders(iii, worker.default_api_url); - const registered = await discoverAndRegister(iii, worker.default_api_url, headers); - return { registered }; - } catch (err) { - // Never throw across the bus boundary — refresh is best-effort. - logger.warn('provider::llamacpp::refresh_models failed', { - err: String(err), - }); - return { registered: [] }; - } - }, - { - description: - 'Re-discover the loaded llama-server model and register it into the iii models catalog. Idempotent.', - }, - ); -} diff --git a/harness/src/provider-llamacpp/register.ts b/harness/src/provider-llamacpp/register.ts deleted file mode 100644 index f2ebce95c..000000000 --- a/harness/src/provider-llamacpp/register.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { loadConfig } from '../runtime/config.js'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import { declareProvider } from '../runtime/provider-resolve.js'; -import { buildAuthHeaders, PROVIDER_ID } from './auth.js'; -import { register as registerComplete } from './complete.js'; -import { loadWorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; -import { register as registerRefresh } from './refresh-fn.js'; -import { register as registerStream } from './stream-fn.js'; - -export async function register(iii: ISdk, ctx: { configPath: string }): Promise { - const cfg = await loadConfig(ctx.configPath); - const worker = loadWorkerConfig(cfg); - registerComplete(iii, worker); - registerStream(iii, worker); - registerRefresh(iii, worker); - - // Self-declare into the harness configuration schema. api_url is - // env-driven (LLAMACPP_BASE_URL); only max_tokens is seeded. - void declareProvider(iii, { - id: PROVIDER_ID, - display_name: 'llama.cpp', - credential_env_var: 'LLAMACPP_API_KEY', - defaults: { max_tokens: worker.default_max_tokens }, - supports_model_listing: true, - }); - - // Fire-and-forget startup discovery: probe llama-server's /v1/models - // and register the (single) loaded model so the picker shows its real - // id instead of just the placeholder. Wrapped in setImmediate so a - // slow/unreachable host doesn't block the rest of the harness from - // coming up — the auth and models-catalog workers may also still be - // registering when this runs. - // - // Note: llama-server CANNOT load/unload models at runtime, so this - // provider does NOT register `provider::llamacpp::load_model` / - // `unload_model`. To run a different model, restart `llama-server` - // with a different `-m`. - setImmediate(() => { - runStartupDiscovery(iii, worker.default_api_url).catch((err) => { - logger.warn('llamacpp startup discovery threw', { err: String(err) }); - }); - }); -} - -async function runStartupDiscovery(iii: ISdk, chatUrl: string): Promise { - try { - const headers = await buildAuthHeaders(iii, chatUrl); - await discoverAndRegister(iii, chatUrl, headers); - } catch (err) { - logger.warn('llamacpp startup discovery: header build failed', { - err: String(err), - }); - } -} diff --git a/harness/src/provider-llamacpp/sse.ts b/harness/src/provider-llamacpp/sse.ts deleted file mode 100644 index 230f4c5d7..000000000 --- a/harness/src/provider-llamacpp/sse.ts +++ /dev/null @@ -1,254 +0,0 @@ -// llama-server SSE handling. Kept separate from the other OpenAI-compat -// providers so llama.cpp-specific quirks (jinja templating modes, -// reasoning-format toggles) can land without coupling the providers. -// -// Compared to provider-lmstudio/sse.ts this file is intentionally -// thinner: there is no "load failure" path because llama-server runs -// exactly one model loaded at process start (via `-m model.gguf`), so -// there is no `provider::llamacpp::load_model` to point the user at if -// classification fails. Errors classify as transient/auth_expired/ -// rate_limited/context_overflow/permanent based on HTTP status alone. - -import type { AssistantMessage } from '../types/agent-message.js'; -import type { ContentBlock } from '../types/content.js'; -import type { AssistantMessageEvent, ErrorKind, StopReason, Usage } from '../types/stream-event.js'; - -type PartialToolCall = { id: string; function_id: string; args_json: string }; - -export type PartialState = { - text: string; - /** - * Accumulated reasoning content from thinking-mode models served by - * llama-server with `--jinja --reasoning-format deepseek` (or - * equivalent). Stream emits these as `delta.reasoning_content`, - * matching the LM Studio / Moonshot convention. Persisted as a - * `thinking` ContentBlock so subsequent requests can echo it back - * via `reasoning_content` — required by some templates when - * thinking is enabled and the assistant turn carries tool_calls. - */ - reasoning_text: string; - tool_calls: PartialToolCall[]; - usage: Usage; - stop_reason: StopReason; - /** - * Set true the first time we observe a chunk carrying a non-null - * `finish_reason`. Used by stream.ts to distinguish a legitimate - * stream end from an abrupt EOF before the server finished. - */ - saw_finish_reason: boolean; -}; - -export function emptyPartial(): PartialState { - return { - text: '', - reasoning_text: '', - tool_calls: [], - usage: { input: 0, output: 0, cache_read: 0, cache_write: 0 }, - stop_reason: 'end', - saw_finish_reason: false, - }; -} - -function buildContent(state: PartialState): ContentBlock[] { - const out: ContentBlock[] = []; - if (state.reasoning_text.length > 0) { - out.push({ type: 'thinking', text: state.reasoning_text }); - } - if (state.text.length > 0) out.push({ type: 'text', text: state.text }); - for (const tc of state.tool_calls) { - if (tc.function_id.length === 0) continue; - let args: unknown = {}; - if (tc.args_json.length > 0) { - try { - args = JSON.parse(tc.args_json); - } catch { - args = null; - } - } - out.push({ type: 'function_call', id: tc.id, function_id: tc.function_id, arguments: args }); - } - return out; -} - -export function buildPartial( - state: PartialState, - model: string, - provider: string, -): AssistantMessage { - return { - role: 'assistant', - content: buildContent(state), - stop_reason: state.stop_reason, - error_message: null, - error_kind: null, - usage: state.usage, - model, - provider, - timestamp: Date.now(), - }; -} - -export function buildFinal(state: PartialState, model: string, provider: string): AssistantMessage { - return buildPartial(state, model, provider); -} - -export function mapFinishReason(s: string): StopReason { - if (s === 'stop') return 'end'; - if (s === 'length') return 'length'; - if (s === 'tool_calls' || s === 'function_call') return 'function_call'; - return 'end'; -} - -export function mergeUsage(usage: Record, into: Usage): void { - const num = (k: string) => (typeof usage[k] === 'number' ? (usage[k] as number) : 0); - into.input = (into.input ?? 0) + num('prompt_tokens') + num('input_tokens'); - into.output = (into.output ?? 0) + num('completion_tokens') + num('output_tokens'); - for (const parent of ['prompt_tokens_details', 'input_tokens_details']) { - const d = usage[parent] as Record | undefined; - if (d && typeof d.cached_tokens === 'number') { - into.cache_read = (into.cache_read ?? 0) + d.cached_tokens; - } - } -} - -export function classifyLlamacppError(message: string, status?: number): ErrorKind { - if (status === 401 || status === 403) return 'auth_expired'; - if (status === 429) return 'rate_limited'; - if (status && status >= 500) return 'transient'; - if (/context length|too many tokens|n_ctx|context window/i.test(message)) { - return 'context_overflow'; - } - return 'permanent'; -} - -export function syntheticErrorEvent( - message: string, - model: string, - provider: string, - error_kind: ErrorKind = 'transient', -): AssistantMessageEvent { - // Carry the message ONLY in `error_message` — same security posture - // as provider-lmstudio (see SECURITY note there). Never inject as a - // `text` ContentBlock that would be re-fed as trusted context. - const final: AssistantMessage = { - role: 'assistant', - content: [], - stop_reason: 'error', - error_message: message, - error_kind, - usage: null, - model, - provider, - timestamp: Date.now(), - }; - return { type: 'error', error: final }; -} - -/** - * Extract a human-readable error message from an SSE error chunk. - * llama-server (and other OpenAI-compatible servers) send a JSON chunk - * shaped `{"error": {"message": "...", ...}}` or `{"error": "..."}` - * when generation fails mid-stream after the HTTP 200 was already - * committed. Returns null when the chunk has no error. - */ -export function extractErrorMessage(chunk: Record): string | null { - const err = chunk.error; - if (!err) return null; - if (typeof err === 'string' && err.length > 0) return err; - if (typeof err === 'object') { - const obj = err as Record; - if (typeof obj.message === 'string' && obj.message.length > 0) return obj.message; - if (typeof obj.error_message === 'string' && obj.error_message.length > 0) { - return obj.error_message; - } - if (typeof obj.detail === 'string' && obj.detail.length > 0) return obj.detail; - } - return null; -} - -export function handleChunk( - chunk: Record, - state: PartialState, - model: string, - provider: string, -): AssistantMessageEvent[] { - const events: AssistantMessageEvent[] = []; - - const errMsg = extractErrorMessage(chunk); - if (errMsg) { - state.stop_reason = 'error'; - state.saw_finish_reason = true; // prevent the EOF-without-finish guard - return [syntheticErrorEvent(errMsg, model, provider, classifyLlamacppError(errMsg))]; - } - - const usage = chunk.usage as Record | undefined; - if (usage) mergeUsage(usage, state.usage); - const choices = chunk.choices; - if (!Array.isArray(choices) || choices.length === 0) return events; - const choice = choices[0] as Record; - const finish = typeof choice.finish_reason === 'string' ? choice.finish_reason : null; - if (finish) { - state.stop_reason = mapFinishReason(finish); - state.saw_finish_reason = true; - } - const delta = choice.delta as Record | undefined; - if (!delta) return events; - - if (typeof delta.reasoning_content === 'string' && delta.reasoning_content.length > 0) { - if (state.reasoning_text.length === 0) { - events.push({ type: 'thinking_start', partial: buildPartial(state, model, provider) }); - } - state.reasoning_text += delta.reasoning_content; - events.push({ - type: 'thinking_delta', - partial: buildPartial(state, model, provider), - delta: delta.reasoning_content, - }); - } - - if (typeof delta.content === 'string' && delta.content.length > 0) { - if (state.text.length === 0) { - events.push({ type: 'text_start', partial: buildPartial(state, model, provider) }); - } - state.text += delta.content; - events.push({ - type: 'text_delta', - partial: buildPartial(state, model, provider), - delta: delta.content, - }); - } - - const tool_calls = delta.tool_calls; - if (Array.isArray(tool_calls)) { - for (const tc of tool_calls) { - if (!tc || typeof tc !== 'object') continue; - const tcObj = tc as Record; - const rawIndex = typeof tcObj.index === 'number' ? tcObj.index : 0; - // DoS guard against attacker-controlled tool-call indices — - // same rationale as provider-lmstudio/sse.ts. - if (!Number.isInteger(rawIndex) || rawIndex < 0 || rawIndex > 256) { - continue; - } - const index = rawIndex; - while (state.tool_calls.length <= index) { - state.tool_calls.push({ id: '', function_id: '', args_json: '' }); - } - const entry = state.tool_calls[index]; - if (!entry) continue; - if (typeof tcObj.id === 'string' && tcObj.id.length > 0) entry.id = tcObj.id; - const fn = tcObj.function as Record | undefined; - if (fn) { - if (typeof fn.name === 'string' && fn.name.length > 0) entry.function_id = fn.name; - if (typeof fn.arguments === 'string') { - entry.args_json += fn.arguments; - events.push({ - type: 'functioncall_delta', - partial: buildPartial(state, model, provider), - delta: fn.arguments, - }); - } - } - } - } - return events; -} diff --git a/harness/src/provider-llamacpp/stream-fn.ts b/harness/src/provider-llamacpp/stream-fn.ts deleted file mode 100644 index 6107b8a7b..000000000 --- a/harness/src/provider-llamacpp/stream-fn.ts +++ /dev/null @@ -1,53 +0,0 @@ -import type { ChannelWriter } from 'iii-sdk'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import { - ProviderStreamInputJsonSchema, - ProviderStreamOutputJsonSchema, - ProviderStreamRuntimeInputSchema, -} from '../types/provider.js'; -import { isTerminal } from '../types/stream-event.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { streamLlamacpp } from './stream.js'; - -export const FUNCTION_ID = 'provider::llamacpp::stream'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (raw: unknown) => { - const input = ProviderStreamRuntimeInputSchema.parse(raw); - const writer = input.writer_ref as ChannelWriter; - const cfg = await buildConfig(iii, worker, input.model); - try { - const events = streamLlamacpp({ - cfg, - system_prompt: input.system_prompt ?? '', - messages: input.messages as AgentMessage[], - tools: input.tools as import('../types/function.js').AgentFunction[], - }); - for await (const ev of events) { - writer.sendMessage(JSON.stringify(ev)); - if (isTerminal(ev)) break; - } - } catch (err) { - logger.warn('provider::llamacpp::stream failed mid-flight', { err: String(err) }); - } finally { - try { - writer.close(); - } catch (err) { - logger.debug('writer.close failed', { err: String(err) }); - } - } - return { ok: true }; - }, - { - description: - 'Stream a single assistant turn from a local llama-server Chat Completions server into the caller-supplied channel.', - request_format: ProviderStreamInputJsonSchema as Record, - response_format: ProviderStreamOutputJsonSchema as Record, - }, - ); -} diff --git a/harness/src/provider-llamacpp/stream.ts b/harness/src/provider-llamacpp/stream.ts deleted file mode 100644 index 37a4035b7..000000000 --- a/harness/src/provider-llamacpp/stream.ts +++ /dev/null @@ -1,294 +0,0 @@ -// llama-server Chat Completions streaming. Kept separate from -// provider-openai / provider-kimi / provider-lmstudio so llama.cpp- -// specific extensions (cache_prompt, slot_id, --jinja knobs) can be -// added without coupling the providers. -// -// Compared to provider-lmstudio/stream.ts this is a leaner generator: -// - No `lmstudio-local` placeholder resolution (llama-server runs one -// model; the model id is whatever the user typed and the server -// accepts any string for the single loaded model). -// - No auto-load retry block. llama-server can't load a different -// model at runtime — it would require restarting the process with -// a different `-m`. -// - All other defenses (timeout, non-2xx truncation, EOF-without- -// finish_reason guard, error event short-circuit) are preserved. - -import { logger } from '../runtime/otel.js'; -import type { AgentMessage, AssistantMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import type { AssistantMessageEvent } from '../types/stream-event.js'; -import { - buildFinal, - classifyLlamacppError, - emptyPartial, - handleChunk, - syntheticErrorEvent, -} from './sse.js'; -import type { ChatCompletionsConfig } from './types.js'; -import { toOpenaiMessages } from './wire-messages.js'; -import { functionsToOpenai } from './wire-tools.js'; - -/** - * Catalog placeholder id. Maps to "whatever llama-server has loaded"; - * the request forwards this string verbatim — llama-server accepts any - * model name for its single loaded model. - */ -export const PLACEHOLDER_MODEL_ID = 'llamacpp-local'; - -/** - * Default connect + first-byte timeout for any llama-server HTTP call. - * - * Generous because large GGUFs (35B+ Q4) can spend tens of seconds on - * prompt processing before the first token arrives — especially when - * llama-server is on a remote LAN host and the prompt is long. A - * too-short timeout surfaces as "RESPONSE FAILED: LLAMACPP FETCH - * TIMED OUT" mid-think. - * - * Override per-machine via `LLAMACPP_FETCH_TIMEOUT_MS` (any positive - * integer in milliseconds; values <1000 are treated as a - * misconfiguration and ignored). - */ -export const DEFAULT_FETCH_TIMEOUT_MS = 120_000; - -export function resolveFetchTimeoutMs(): number { - const raw = (process.env.LLAMACPP_FETCH_TIMEOUT_MS ?? '').trim(); - if (raw.length === 0) return DEFAULT_FETCH_TIMEOUT_MS; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed < 1_000) return DEFAULT_FETCH_TIMEOUT_MS; - return Math.floor(parsed); -} - -export type StreamArgs = { - cfg: ChatCompletionsConfig; - system_prompt: string; - messages: AgentMessage[]; - tools: AgentFunction[]; -}; - -/** - * Run `fetch` with an AbortController-based timeout. Without this a - * misconfigured URL hangs for ~75s (macOS SYN timeout) and the UI - * stays stuck on "thinking…" the whole time. - */ -async function fetchWithTimeout( - url: string, - init: RequestInit, - timeoutMs: number, -): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), timeoutMs); - try { - return await fetch(url, { ...init, signal: controller.signal }); - } finally { - clearTimeout(timer); - } -} - -/** - * Strip C0 control chars (except TAB / LF / CR) and DEL. Mirrors - * provider-lmstudio's helper. Written as a charCodeAt loop rather - * than a regex literal so Biome's `noControlCharactersInRegex` is - * happy. - */ -function stripControlChars(s: string): string { - let out = ''; - for (let i = 0; i < s.length; i++) { - const code = s.charCodeAt(i); - if (code < 32 && code !== 9 && code !== 10 && code !== 13) continue; - if (code === 127) continue; - out += s[i]; - } - return out; -} - -export async function* streamLlamacpp({ - cfg, - system_prompt, - messages, - tools, -}: StreamArgs): AsyncGenerator { - const authName = cfg.auth_header_name ?? 'Authorization'; - const authPrefix = cfg.auth_value_prefix ?? 'Bearer '; - const headers: Record = { - 'content-type': 'application/json', - }; - if (cfg.api_key.length > 0) { - headers[authName] = `${authPrefix}${cfg.api_key}`; - } - for (const [k, v] of cfg.extra_headers ?? []) headers[k] = v; - - const effectiveModel = cfg.model; - const body: Record = { - model: effectiveModel, - max_completion_tokens: cfg.max_tokens, - messages: toOpenaiMessages(messages, system_prompt), - stream: true, - stream_options: { include_usage: true }, - }; - if (tools.length > 0) body.tools = functionsToOpenai(tools); - - // Resolve the timeout fresh on each call so tests can mutate the - // env var between cases, and so an operator who sets - // `LLAMACPP_FETCH_TIMEOUT_MS` at runtime (e.g. via a config - // reload) sees the new value without restarting the worker. - const fetchTimeoutMs = resolveFetchTimeoutMs(); - let resp: Response; - try { - resp = await fetchWithTimeout( - cfg.url, - { method: 'POST', headers, body: JSON.stringify(body) }, - fetchTimeoutMs, - ); - } catch (err) { - const msg = - err instanceof Error && err.name === 'AbortError' - ? `llamacpp fetch timed out after ${fetchTimeoutMs / 1000}s (is ${cfg.url} reachable?)` - : `llamacpp fetch failed: ${String(err)}`; - yield syntheticErrorEvent(msg, effectiveModel, cfg.provider_name); - return; - } - if (!resp.ok) { - const text = await resp.text().catch(() => ''); - // Truncate + strip control chars before surfacing — non-2xx - // responses can carry HTML, proxy error pages, or attacker- - // controlled bodies. Length cap keeps the message readable; - // control-char strip avoids ANSI / newline injection. - const safeText = stripControlChars(text).slice(0, 256); - yield syntheticErrorEvent( - safeText || `llamacpp http ${resp.status}`, - cfg.model, - cfg.provider_name, - classifyLlamacppError(safeText, resp.status), - ); - return; - } - const partial: AssistantMessage = { - role: 'assistant', - content: [], - stop_reason: 'end', - error_message: null, - error_kind: null, - usage: null, - model: effectiveModel, - provider: cfg.provider_name, - timestamp: Date.now(), - }; - yield { type: 'start', partial }; - - const state = emptyPartial(); - if (!resp.body) { - yield syntheticErrorEvent('llamacpp response missing body', effectiveModel, cfg.provider_name); - return; - } - const reader = resp.body.getReader(); - const decoder = new TextDecoder(); - let buf = ''; - let parsedChunkCount = 0; - try { - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - let idx = buf.indexOf('\n\n'); - while (idx >= 0) { - const block = buf.slice(0, idx); - buf = buf.slice(idx + 2); - const dataLine = parseDataLine(block); - idx = buf.indexOf('\n\n'); - if (dataLine === null) continue; - if (dataLine === '[DONE]') { - yield { - type: 'done', - message: buildFinal(state, effectiveModel, cfg.provider_name), - }; - return; - } - let parsed: Record | null = null; - try { - parsed = JSON.parse(dataLine) as Record; - } catch { - continue; - } - if (parsed) { - parsedChunkCount++; - for (const e of handleChunk(parsed, state, effectiveModel, cfg.provider_name)) { - yield e; - // handleChunk returns an `error` event when llama-server - // sent an SSE error chunk. Stop reading immediately so - // the rest of the body (typically just a connection - // close) doesn't fall into the EOF-without-finish_reason - // guard and overwrite the server's specific error. - if (e.type === 'error') return; - } - } - } - } - } catch (err) { - logger.warn('llamacpp stream read failed', { err: String(err) }); - yield syntheticErrorEvent( - `stream read failed: ${String(err)}`, - effectiveModel, - cfg.provider_name, - ); - return; - } - if (parsedChunkCount === 0) { - yield syntheticErrorEvent( - 'llamacpp returned a 200 response with a non-SSE body (no parseable chunks). The endpoint URL is likely wrong, or the server served an HTML page in place of an SSE stream.', - cfg.model, - cfg.provider_name, - ); - return; - } - // We parsed some chunks but never saw `data: [DONE]` AND never saw a - // `finish_reason`. The server closed the body mid-response — typical - // causes: GPU OOM, context exhausted, host disconnect. Promote to an - // explicit error so the UI doesn't show a silently truncated reply. - if (!state.saw_finish_reason) { - const partial = buildFinal(state, effectiveModel, cfg.provider_name); - const tokenHint = - partial.usage && (partial.usage.output ?? 0) > 0 - ? ` after ~${partial.usage.output} output tokens` - : ''; - yield syntheticErrorEvent( - `llamacpp stream closed mid-response${tokenHint} — the server ended the SSE body without a [DONE] marker or finish_reason. Common causes: GPU OOM, host disconnect, or context exhausted during generation.`, - effectiveModel, - cfg.provider_name, - 'transient', - ); - return; - } - yield { type: 'done', message: buildFinal(state, effectiveModel, cfg.provider_name) }; -} - -function parseDataLine(block: string): string | null { - let data: string | null = null; - for (const line of block.split('\n')) { - if (line.startsWith('data: ')) data = line.slice('data: '.length); - } - return data; -} - -export async function collect( - events: AsyncIterable, -): Promise { - let last: AssistantMessage | null = null; - for await (const ev of events) { - if (ev.type === 'done') return ev.message; - if (ev.type === 'error') return ev.error; - if ('partial' in ev) last = ev.partial; - } - return ( - last ?? { - role: 'assistant', - content: [], - stop_reason: 'error', - error_message: 'stream closed without final', - error_kind: 'transient', - usage: null, - model: 'llamacpp', - provider: 'llamacpp', - timestamp: Date.now(), - } - ); -} diff --git a/harness/src/provider-llamacpp/types.ts b/harness/src/provider-llamacpp/types.ts deleted file mode 100644 index 145e9a3e2..000000000 --- a/harness/src/provider-llamacpp/types.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { Credential } from '../runtime/provider-resolve.js'; - -export type ChatCompletionsConfig = { - url: string; - provider_name: string; - model: string; - /** Empty string when llama-server is running without --api-key (the default). */ - api_key: string; - /** Defaults to "Authorization". */ - auth_header_name?: string; - /** Defaults to "Bearer ". */ - auth_value_prefix?: string; - extra_headers?: Array; - max_tokens: number; -}; - -export function configFromCredential( - url: string, - provider_name: string, - model: string, - cred: Credential | null, - max_tokens: number, -): ChatCompletionsConfig { - const api_key = cred === null ? '' : cred.type === 'api_key' ? cred.key : cred.access_token; - return { url, provider_name, model, api_key, max_tokens }; -} diff --git a/harness/src/provider-llamacpp/wire-messages.ts b/harness/src/provider-llamacpp/wire-messages.ts deleted file mode 100644 index f0f054f6a..000000000 --- a/harness/src/provider-llamacpp/wire-messages.ts +++ /dev/null @@ -1,117 +0,0 @@ -// llama-server speaks OpenAI's Chat Completions wire format. Kept -// separate from provider-openai / provider-kimi / provider-lmstudio so -// any llama.cpp-specific extensions (e.g. `cache_prompt`, `slot_id`, -// `--jinja` template knobs) can land without coupling the providers. -// -// Mirrors provider-lmstudio/wire-messages.ts closely — same jinja -// template constraints apply because both stacks run the same GGUF -// models through compatible templates. - -import { logger } from '../runtime/otel.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import { formatFunctionResultContent } from '../types/wire.js'; - -/** - * Strict jinja templates (notably qwen3's) require at least one - * `role: 'user'` message and abort with "No user query found in - * messages" otherwise. This condition is reachable in normal agentic - * operation: after a tool-call cycle whose ancestors were summarised - * away by async compaction, the flat-state can be [assistant, tool, - * assistant, tool, …] with the original user message gone. Without - * this safety net we'd send the request, llama-server would error - * mid-stream, and the user would see a cryptic "stream closed - * mid-response" instead of a clean continuation. - * - * The placeholder is intentionally minimal ("(continue)") so the - * model understands it as a continuation directive — matches what - * LiteLLM / OpenRouter do for the same template constraint. - */ -export const PLACEHOLDER_USER_MESSAGE = '(continue)'; - -function hasUserMessage(wire: readonly unknown[]): boolean { - for (const m of wire) { - if (m && typeof m === 'object' && (m as { role?: unknown }).role === 'user') { - return true; - } - } - return false; -} - -export function toOpenaiMessages(messages: AgentMessage[], system_prompt: string): unknown[] { - const out: unknown[] = []; - // O(1) latest-wins dedup of function_result rows. Same approach as - // provider-lmstudio/wire-messages.ts — see that file for the - // history-amplification reasoning. - const toolResultIndexById = new Map(); - if (system_prompt.length > 0) { - out.push({ role: 'system', content: system_prompt }); - } - for (const m of messages) { - if (m.role === 'user') { - const text = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'text' }> => c.type === 'text', - ) - .map((c) => c.text) - .join('\n'); - out.push({ role: 'user', content: text }); - } else if (m.role === 'assistant') { - const text = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'text' }> => c.type === 'text', - ) - .map((c) => c.text) - .join('\n'); - // Thinking-mode models served via `--jinja --reasoning-format - // deepseek` emit `reasoning_content` on assistant tool-call - // messages and expect it echoed back. Captured into a ThinkingContent - // block by sse.ts; projected back here. - const reasoning = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'thinking' }> => - c.type === 'thinking', - ) - .map((c) => c.text) - .join(''); - const tool_calls = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'function_call' }> => - c.type === 'function_call', - ) - .map((c) => ({ - id: c.id, - type: 'function', - function: { name: c.function_id, arguments: JSON.stringify(c.arguments) }, - })); - const entry: Record = { role: 'assistant' }; - if (reasoning.length > 0) entry.reasoning_content = reasoning; - if (text.length > 0) entry.content = text; - if (tool_calls.length > 0) entry.tool_calls = tool_calls; - out.push(entry); - } else if (m.role === 'function_result') { - const text = formatFunctionResultContent(m); - const row: Record = { - role: 'tool', - tool_call_id: m.function_call_id, - content: text, - }; - if (m.is_error) row.is_error = true; - const existingIdx = toolResultIndexById.get(m.function_call_id); - if (existingIdx !== undefined) { - out[existingIdx] = row; - } else { - toolResultIndexById.set(m.function_call_id, out.length); - out.push(row); - } - } - // custom messages are skipped - } - if (!hasUserMessage(out)) { - logger.warn( - 'llamacpp: no user-role message in request; injecting placeholder to satisfy strict jinja templates (e.g. qwen3 "No user query found")', - { messageCount: out.length }, - ); - out.push({ role: 'user', content: PLACEHOLDER_USER_MESSAGE }); - } - return out; -} diff --git a/harness/src/provider-llamacpp/wire-tools.ts b/harness/src/provider-llamacpp/wire-tools.ts deleted file mode 100644 index c133e4f33..000000000 --- a/harness/src/provider-llamacpp/wire-tools.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { AgentFunction } from '../types/function.js'; - -export function functionsToOpenai(functions: AgentFunction[]): unknown[] { - return functions.map((t) => ({ - type: 'function', - function: { - name: t.name, - description: t.description, - parameters: t.parameters, - }, - })); -} diff --git a/harness/src/provider-lmstudio/auth.ts b/harness/src/provider-lmstudio/auth.ts deleted file mode 100644 index e64fea93e..000000000 --- a/harness/src/provider-lmstudio/auth.ts +++ /dev/null @@ -1,166 +0,0 @@ -import type { ISdk } from '../runtime/iii.js'; -import { normalizeChatCompletionsUrl } from '../runtime/openai-compat-url.js'; -import { logger } from '../runtime/otel.js'; -import { clampOutputTokens, getCatalogModel } from '../runtime/output-tokens.js'; -import { - type Credential, - type ProviderResolveResult, - resolveProvider, -} from '../runtime/provider-resolve.js'; -import type { WorkerConfig } from './config.js'; -import { type ChatCompletionsConfig, configFromCredential } from './types.js'; - -// LM Studio is local-first: by default the localhost REST server runs without -// authentication and ignores the Authorization header. We still go through -// the harness provider registry so users who run an authenticated LM Studio -// deployment can opt-in via the configured api key (or `LMSTUDIO_API_KEY`), -// but when the credential is missing or empty AND the URL points at loopback -// we fall back to the literal string `"lm-studio"` (LM Studio's own -// convention — see https://lmstudio.ai/docs/local-server). For non-loopback -// hosts we refuse to send a fallback bearer: a misconfigured LMSTUDIO_BASE_URL -// (e.g. via a stale EnvironmentFile, a tunnel host, an ssh-forward) would -// otherwise leak a recognisable provider fingerprint to whoever is on the -// other end. -export const PROVIDER_ID = 'lmstudio'; -const FALLBACK_API_KEY = 'lm-studio'; - -const EMPTY_RESOLVE: ProviderResolveResult = { - configured: false, - source: null, - credential: null, - api_url: null, - max_tokens: null, -}; - -function extractKey(cred: Credential | null): string { - if (!cred) return ''; - return cred.type === 'api_key' ? cred.key : cred.access_token; -} - -/** - * `true` when `url` resolves to a localhost / loopback target. Used to - * decide whether the fallback bearer is safe to send. - */ -export function isLoopbackUrl(url: string): boolean { - let parsed: URL; - try { - parsed = new URL(url); - } catch { - return false; - } - const host = parsed.hostname.toLowerCase(); - if (host === 'localhost') return true; - if (host === '127.0.0.1' || host === '::1' || host === '[::1]') return true; - if (host.endsWith('.localhost')) return true; - // 127.0.0.0/8 — IPv4 loopback block - if (/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host)) return true; - return false; -} - -/** - * Resolve this provider's credential + settings via the harness registry. - * Tolerant: LM Studio is a normal localhost-no-auth setup, so a missing - * harness/registry yields an empty result rather than throwing. Logs at - * WARN with a stable code so monitoring can alert on a sustained - * fallback-key rate. - */ -async function resolveTolerant(iii: ISdk): Promise { - try { - return await resolveProvider(iii, PROVIDER_ID); - } catch (err) { - logger.warn('lmstudio.auth: resolve failed; falling back to no-credential', { - code: 'lmstudio_auth_fetch_failed', - err: String(err), - }); - return EMPTY_RESOLVE; - } -} - -export async function fetchCredential(iii: ISdk): Promise { - return (await resolveTolerant(iii)).credential; -} - -/** - * Decide which API key (if any) to send for `url`. - * - * - Explicit credential present → use it. - * - Otherwise loopback → send the LM Studio fallback ("lm-studio"). - * - Otherwise (non-loopback, no explicit credential) → null, meaning - * omit the Authorization header entirely. Sending a fallback bearer - * to an arbitrary host can leak the provider fingerprint to whoever - * operates that host. - */ -export function selectAuthKey(cred: Credential | null, url: string): string | null { - const key = extractKey(cred); - if (key.length > 0) return key; - if (isLoopbackUrl(url)) return FALLBACK_API_KEY; - logger.warn( - 'lmstudio.auth: no credential configured AND LMSTUDIO_BASE_URL is non-loopback; omitting Authorization header', - { - code: 'lmstudio_auth_omitted_nonloopback', - // Don't log the full URL — it might carry tokens in query - // params or path. Just the origin (scheme + host + port). - origin: (() => { - try { - const u = new URL(url); - return `${u.protocol}//${u.host}`; - } catch { - return ''; - } - })(), - }, - ); - return null; -} - -export async function buildConfig( - iii: ISdk, - worker: WorkerConfig, - model: string, -): Promise { - const resolved = await resolveTolerant(iii); - const cred = resolved.credential; - // Normalise the override URL so a base-URL save from the config UI - // (e.g. `http://host:1234`) gets `/v1/chat/completions` appended. - // worker.default_api_url is already normalised in resolveApiUrl. - const overrideUrl = resolved.api_url ? normalizeChatCompletionsUrl(resolved.api_url) : null; - const apiUrl = overrideUrl ?? worker.default_api_url; - const catalog = await getCatalogModel(iii, PROVIDER_ID, model); - const maxTokens = clampOutputTokens({ - modelMaxOutput: catalog?.max_output_tokens, - userOverride: resolved.max_tokens, - workerDefault: worker.default_max_tokens, - }); - const key = selectAuthKey(cred, apiUrl); - // configFromCredential expects a Credential. When no key is - // available we still pass a synthetic api_key with the LM Studio - // fallback string so the downstream interface stays uniform; the - // calling code (stream.ts / discover.ts) goes through - // buildAuthHeaders for the actual header emission, where the - // null-key case omits the Authorization header. - const effective: Credential = - key !== null - ? cred && extractKey(cred).length > 0 - ? (cred as Credential) - : { type: 'api_key', key } - : { type: 'api_key', key: FALLBACK_API_KEY }; - return configFromCredential(apiUrl, 'lmstudio', model, effective, maxTokens); -} - -/** - * Build the HTTP headers used for any LM Studio REST call (chat - * completions AND `/api/v0/models` discovery). Shared so the auth - * dance — credential lookup, loopback-vs-remote decision — lives in - * exactly one place. - */ -export async function buildAuthHeaders(iii: ISdk, url: string): Promise> { - const cred = await fetchCredential(iii); - const token = selectAuthKey(cred, url); - const base: Record = { - 'content-type': 'application/json', - }; - if (token !== null) { - base.Authorization = `Bearer ${token}`; - } - return base; -} diff --git a/harness/src/provider-lmstudio/complete.ts b/harness/src/provider-lmstudio/complete.ts deleted file mode 100644 index 5156ed537..000000000 --- a/harness/src/provider-lmstudio/complete.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { requireString } from '../runtime/handler.js'; -import type { ISdk } from '../runtime/iii.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { collect, streamLmstudio } from './stream.js'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - 'provider::lmstudio::complete', - async (payload: unknown) => { - const obj = (payload ?? {}) as Record; - const model = requireString(obj, 'model'); - const system_prompt = typeof obj.system_prompt === 'string' ? obj.system_prompt : ''; - const messages = Array.isArray(obj.messages) ? (obj.messages as AgentMessage[]) : []; - const tools = Array.isArray(obj.tools) ? (obj.tools as AgentFunction[]) : []; - const cfg = await buildConfig(iii, worker, model); - return await collect(streamLmstudio({ cfg, system_prompt, messages, tools })); - }, - { - description: - 'Legacy: drain a streamed LM Studio chat-completion and return the final AssistantMessage.', - }, - ); -} diff --git a/harness/src/provider-lmstudio/config.ts b/harness/src/provider-lmstudio/config.ts deleted file mode 100644 index 4cd9c6a7f..000000000 --- a/harness/src/provider-lmstudio/config.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { logger } from '../runtime/otel.js'; -import { getNumber, getSection, getString } from '../runtime/config.js'; - -export type WorkerConfig = { - default_max_tokens: number; - default_api_url: string; -}; - -export const DEFAULT_API_URL = 'http://localhost:1234/v1/chat/completions'; - -/** - * Validate a candidate URL: parses cleanly AND uses an http/https - * scheme. Returns the parsed URL if valid, null otherwise. - * - * Pre-fix, `LMSTUDIO_BASE_URL` was concatenated into the request - * target with no validation — a malformed or attacker-controlled - * value (stale EnvironmentFile, leaked from a parent process, typo - * with extra leading scheme) would silently route the bearer token - * to that host. - */ -function validatedUrl(raw: string): URL | null { - let parsed: URL; - try { - parsed = new URL(raw); - } catch { - return null; - } - if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { - return null; - } - return parsed; -} - -function isLoopbackHost(host: string): boolean { - const h = host.toLowerCase(); - if (h === 'localhost' || h.endsWith('.localhost')) return true; - if (h === '127.0.0.1' || h === '::1' || h === '[::1]') return true; - if (/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(h)) return true; - return false; -} - -/** - * Resolve the LM Studio base URL with this precedence: - * 1. `LMSTUDIO_BASE_URL` env var (per-machine override — wins over yaml) - * 2. `provider_lmstudio.default_api_url` in config.yaml - * 3. The localhost DEFAULT_API_URL constant above - * - * The env var accepts either a base origin (`http://host:port`) or a - * full URL (`http://host:port/v1/chat/completions`). When only a base - * is given, `/v1/chat/completions` is appended automatically. - * - * Both the env value and the yaml value are validated as http(s) URLs; - * a malformed value falls back to the next tier rather than being - * concatenated raw. A warning is logged when the resolved host is not - * loopback so operators see they're shipping their bearer to a remote. - */ -function resolveApiUrl(yamlValue: string): string { - const envRaw = (process.env.LMSTUDIO_BASE_URL ?? '').trim(); - let candidate: string | null = null; - if (envRaw.length > 0) { - const trimmedEnv = envRaw.endsWith('/') ? envRaw.slice(0, -1) : envRaw; - const withPath = trimmedEnv.includes('/chat/completions') - ? trimmedEnv - : `${trimmedEnv}/v1/chat/completions`; - if (validatedUrl(withPath)) { - candidate = withPath; - } else { - logger.warn('lmstudio.config: LMSTUDIO_BASE_URL is not a valid http(s) URL — ignoring', { - code: 'lmstudio_base_url_invalid', - }); - } - } - if (candidate === null) candidate = yamlValue; - const parsed = validatedUrl(candidate); - if (!parsed) { - logger.warn( - 'lmstudio.config: resolved API URL is not a valid http(s) URL; falling back to default', - { code: 'lmstudio_api_url_invalid' }, - ); - return DEFAULT_API_URL; - } - if (!isLoopbackHost(parsed.hostname)) { - logger.warn( - 'lmstudio.config: API URL is non-loopback — bearer (if configured) will be sent to a remote host', - { - code: 'lmstudio_api_url_remote', - origin: `${parsed.protocol}//${parsed.host}`, - }, - ); - } - return candidate; -} - -export function loadWorkerConfig(cfg: Record): WorkerConfig { - const section = getSection(cfg, 'provider_lmstudio'); - const yamlUrl = getString(section, 'default_api_url', DEFAULT_API_URL); - return { - default_max_tokens: getNumber(section, 'default_max_tokens', 8192), - default_api_url: resolveApiUrl(yamlUrl), - }; -} diff --git a/harness/src/provider-lmstudio/discover.ts b/harness/src/provider-lmstudio/discover.ts deleted file mode 100644 index 644168b24..000000000 --- a/harness/src/provider-lmstudio/discover.ts +++ /dev/null @@ -1,249 +0,0 @@ -/** - * LM Studio model discovery — hits the native `GET /api/v0/models` endpoint - * and registers each loaded LLM into the iii models catalog so the - * picker/dropdown shows them by their real IDs (e.g. `qwen/qwen3-4b-2507`) - * instead of just the `lmstudio-local` placeholder. - * - * Why the native v0 endpoint and not OpenAI-compatible `/v1/models`: - * - v0 returns `state` (loaded / not-loaded / loading) so we can filter - * to only models that are actually serving right now. - * - v0 returns `type` (llm / vlm / embeddings) so we skip non-chat models. - * - v0 returns `loaded_context_length` / `max_context_length` so the - * catalog row carries the real context window. - * - * Best-effort: all failures (LM Studio offline, malformed JSON, register - * RPC errors) are logged and swallowed — the worker still boots, and the - * embedded `lmstudio-local` placeholder remains usable as a fallback. - */ - -import type { Model } from '../models-catalog/types.js'; -import type { ISdk } from '../runtime/iii.js'; -import { reconcileModels } from '../runtime/models-discovery.js'; -import { logger } from '../runtime/otel.js'; -import { PROVIDER_ID } from './auth.js'; - -/** Timeout for the discovery HTTP call. Short — LM Studio is local. */ -const DISCOVERY_TIMEOUT_MS = 5_000; - -/** Defaults for fields LM Studio doesn't expose directly. */ -const DEFAULT_CONTEXT_WINDOW = 32_768; -const DEFAULT_MAX_OUTPUT_TOKENS = 8_192; - -/** - * LM Studio's `/api/v0/models` per-entry shape. Parsed defensively so the - * code keeps working if LM Studio adds or renames fields between versions. - */ -type LmstudioModel = { - id?: unknown; - type?: unknown; - state?: unknown; - arch?: unknown; - quantization?: unknown; - publisher?: unknown; - max_context_length?: unknown; - loaded_context_length?: unknown; -}; - -type LmstudioModelsResponse = { - data?: unknown; -}; - -async function fetchWithTimeout( - url: string, - init: RequestInit, - timeoutMs: number, -): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), timeoutMs); - try { - return await fetch(url, { ...init, signal: controller.signal }); - } finally { - clearTimeout(timer); - } -} - -/** - * Derive the native `/api/v0/models` endpoint from a chat-completions URL. - * - * Handles both URL forms the worker can be configured with: - * - OpenAI-compatible: `http://host:port/v1/chat/completions` - * - LM Studio native: `http://host:port/api/v0/chat/completions` - * - * Anything else (custom proxy path) falls through to `/api/v0/models` - * by appending — the caller's URL is preserved. - */ -export function nativeModelsUrl(chatUrl: string): string { - const trimmed = chatUrl.replace(/\/(v1|api\/v\d+)\/chat\/completions\/?$/, ''); - return `${trimmed}/api/v0/models`; -} - -function asString(v: unknown): string | null { - return typeof v === 'string' && v.length > 0 ? v : null; -} - -function asPositiveNumber(v: unknown): number | null { - return typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : null; -} - -/** - * Convert one LM Studio model entry into a catalog `Model`. - * - * Returns `null` for entries that aren't usable LLMs (no id, embeddings, - * or any non-llm/non-vlm type LM Studio surfaces). - */ -export function toCatalogModel(m: LmstudioModel): Model | null { - const id = asString(m.id); - if (!id) return null; - const type = asString(m.type); - // `llm` = text models; `vlm` = vision-language; anything else (embedding, - // audio, ...) the chat orchestrator doesn't route to. - if (type !== null && type !== 'llm' && type !== 'vlm') return null; - const context_window = - asPositiveNumber(m.loaded_context_length) ?? - asPositiveNumber(m.max_context_length) ?? - DEFAULT_CONTEXT_WINDOW; - return { - id, - provider: 'lmstudio', - api: 'openai-completions', - display_name: id, - context_window, - max_output_tokens: DEFAULT_MAX_OUTPUT_TOKENS, - supports_thinking: false, - supports_xhigh: false, - supports_tools: true, - supports_vision: type === 'vlm', - supports_cache: false, - transports: ['sse'], - }; -} - -/** - * Raw fetch of `/api/v0/models`. Returns the parsed `data` array verbatim - * (no filtering, no catalog mapping) so callers can pick which subset they - * need: loaded only, downloaded, by id, etc. Returns `[]` on any error. - */ -async function fetchRawModels( - chatUrl: string, - headers: Record, -): Promise { - const url = nativeModelsUrl(chatUrl); - let resp: Response; - try { - resp = await fetchWithTimeout(url, { method: 'GET', headers }, DISCOVERY_TIMEOUT_MS); - } catch (err) { - logger.warn('lmstudio discovery: fetch failed', { url, err: String(err) }); - return []; - } - if (!resp.ok) { - logger.warn('lmstudio discovery: non-2xx response', { - url, - status: resp.status, - }); - return []; - } - let parsed: LmstudioModelsResponse; - try { - parsed = (await resp.json()) as LmstudioModelsResponse; - } catch (err) { - logger.warn('lmstudio discovery: invalid JSON', { url, err: String(err) }); - return []; - } - return Array.isArray(parsed.data) ? (parsed.data as LmstudioModel[]) : []; -} - -/** - * Fetch `/api/v0/models` and return the catalog `Model` for each LLM/VLM - * that is *currently loaded*. Used by the chat stream handler when the - * picker sent the `lmstudio-local` placeholder — resolves it to whatever - * is loaded right now. - * - * Returns `[]` on any error or when no LLMs are currently loaded. - */ -export async function discoverLoadedModels( - chatUrl: string, - headers: Record, -): Promise { - const entries = await fetchRawModels(chatUrl, headers); - return entries - .filter((m) => asString(m.state) === 'loaded') - .map(toCatalogModel) - .filter((m): m is Model => m !== null); -} - -/** - * Fetch `/api/v0/models` and return the catalog `Model` for *every* - * downloaded LLM/VLM, loaded or not. Used by startup discovery so the - * picker shows all the user's available models — they can pick one - * that's not loaded yet and the stream handler will auto-load it. - */ -export async function discoverAllDownloadedModels( - chatUrl: string, - headers: Record, -): Promise { - const entries = await fetchRawModels(chatUrl, headers); - return entries.map(toCatalogModel).filter((m): m is Model => m !== null); -} - -/** - * Fetch `/api/v0/models` and return a `Set` of model IDs that are - * currently in `state == "loaded"`. Cheap helper used by the stream - * handler to decide whether it needs to call the load endpoint before - * sending the chat request. - */ -export async function discoverLoadedIds( - chatUrl: string, - headers: Record, -): Promise> { - const entries = await fetchRawModels(chatUrl, headers); - const ids = new Set(); - for (const m of entries) { - if (asString(m.state) !== 'loaded') continue; - const id = asString(m.id); - if (id) ids.add(id); - } - return ids; -} - -/** Register discovered models in one `models::reconcile` call. */ -export async function registerDiscovered(iii: ISdk, models: readonly Model[]): Promise { - if (models.length === 0) return []; - const provider = models[0]?.provider; - if (!provider) return []; - return reconcileModels(iii, provider, models); -} - -/** - * One-shot: discover every downloaded LM Studio LLM/VLM and register each - * into the iii models catalog. Used at worker startup (fire-and-forget) - * and by the `provider::lmstudio::refresh_models` bus function on demand. - * - * We register *all* downloaded models, not just the loaded ones, so the - * picker shows everything the user could pick. The stream handler - * auto-loads not-yet-loaded models on first use (see stream.ts). - */ -export async function discoverAndRegister( - iii: ISdk, - chatUrl: string, - headers: Record, -): Promise { - const models = await discoverAllDownloadedModels(chatUrl, headers); - if (models.length === 0) { - // Empty can mean "server offline" or "nothing downloaded" — we can't tell - // them apart here, so keep the last-known catalog rather than risk wiping - // it on a transient blip. - logger.info('lmstudio discovery: no downloaded LLMs found', {}); - return []; - } - const registered = await reconcileModels(iii, PROVIDER_ID, models); - // Log count at INFO; full id list at DEBUG so model identifiers - // (which may carry fine-tune / org context) don't leak into - // shared observability tooling on the cheaper tier. - logger.info('lmstudio discovery: registered models', { - count: registered.length, - }); - logger.debug('lmstudio discovery: registered model ids', { - ids: registered, - }); - return registered; -} diff --git a/harness/src/provider-lmstudio/iii.worker.yaml b/harness/src/provider-lmstudio/iii.worker.yaml deleted file mode 100644 index 873a57ae2..000000000 --- a/harness/src/provider-lmstudio/iii.worker.yaml +++ /dev/null @@ -1,17 +0,0 @@ -iii: v1 -name: provider-lmstudio -language: node -deploy: binary -manifest: package.json -bin: iii-provider-lmstudio -description: LM Studio (localhost) Chat Completions streaming provider; exposes provider::lmstudio::stream and provider::lmstudio::complete on the iii bus. - -runtime: - kind: node - -scripts: - install: pnpm install - start: node ./dist/provider-lmstudio/main.js --config ./config.yaml - -dependencies: - configuration: "^0.11.0" diff --git a/harness/src/provider-lmstudio/load-fn.ts b/harness/src/provider-lmstudio/load-fn.ts deleted file mode 100644 index 5b492e912..000000000 --- a/harness/src/provider-lmstudio/load-fn.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * `provider::lmstudio::load_model` — bus function the UI / orchestrator - * calls to load a model into LM Studio's memory on demand. Wraps the - * native `POST /api/v1/models/load` endpoint. - * - * Typical use: - * - UI calls this when the user picks a model from the dropdown that - * isn't yet loaded, so streaming starts immediately afterwards. - * - The stream handler also auto-calls the underlying loader on first - * use for a not-yet-loaded model; this bus function is the explicit - * "preload now" knob. - * - * Returns the LM Studio load result on success, or - * `{ ok: false, error: string }` on failure — never throws across the - * bus boundary so a UI prefetch failure doesn't crash the picker. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import { buildAuthHeaders } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { type LoadOptions, type LoadResult, loadModel } from './load.js'; - -export const FUNCTION_ID = 'provider::lmstudio::load_model'; - -export type LoadModelPayload = { - model: string; -} & LoadOptions; - -export type LoadModelResult = LoadResult | { ok: false; error: string }; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (raw: unknown): Promise => { - const payload = raw as Partial | null | undefined; - if (!payload || typeof payload.model !== 'string' || payload.model.length === 0) { - return { ok: false, error: 'invalid payload: { model: string } is required' }; - } - try { - const headers = await buildAuthHeaders(iii, worker.default_api_url); - return await loadModel(worker.default_api_url, headers, payload.model, { - context_length: payload.context_length, - eval_batch_size: payload.eval_batch_size, - flash_attention: payload.flash_attention, - num_experts: payload.num_experts, - offload_kv_cache_to_gpu: payload.offload_kv_cache_to_gpu, - echo_load_config: payload.echo_load_config, - }); - } catch (err) { - logger.warn('provider::lmstudio::load_model failed', { - model: payload.model, - err: String(err), - }); - return { ok: false, error: String(err) }; - } - }, - { - description: - 'Load an LM Studio model into memory via POST /api/v1/models/load. Blocks until ready (up to 120s).', - }, - ); -} diff --git a/harness/src/provider-lmstudio/load.ts b/harness/src/provider-lmstudio/load.ts deleted file mode 100644 index b22119454..000000000 --- a/harness/src/provider-lmstudio/load.ts +++ /dev/null @@ -1,162 +0,0 @@ -/** - * HTTP helpers for LM Studio's native v1 model management endpoints: - * - `POST /api/v1/models/load` — block until a model is in memory - * - `POST /api/v1/models/unload` — release a loaded instance - * - * These are LM Studio 0.4+ only (the v1 native API). They're not exposed - * on the legacy `/v1/*` OpenAI-compatible path or the `/api/v0/*` native - * path. Callers that hit older LM Studio versions will get a 404; the - * stream handler tolerates that (JIT can still kick in, or the chat call - * will surface a clear error). - */ - -import { logger } from '../runtime/otel.js'; - -/** Load can take a while for large models on consumer hardware. */ -const LOAD_TIMEOUT_MS = 120_000; -/** Unload is fast — just frees memory. */ -const UNLOAD_TIMEOUT_MS = 10_000; - -/** All optional knobs the load endpoint accepts. */ -export type LoadOptions = { - context_length?: number; - eval_batch_size?: number; - flash_attention?: boolean; - num_experts?: number; - offload_kv_cache_to_gpu?: boolean; - /** When true, the response includes the resolved `load_config`. */ - echo_load_config?: boolean; -}; - -/** Shape of a successful load response. */ -export type LoadResult = { - type: 'llm' | 'embedding' | string; - instance_id: string; - load_time_seconds: number; - status: 'loaded' | string; - load_config?: Record; -}; - -/** Shape of a successful unload response. */ -export type UnloadResult = { - instance_id: string; -}; - -async function fetchWithTimeout( - url: string, - init: RequestInit, - timeoutMs: number, -): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), timeoutMs); - try { - return await fetch(url, { ...init, signal: controller.signal }); - } finally { - clearTimeout(timer); - } -} - -/** - * Derive `/api/v1/models/load` from any chat-completions URL we accept. - * Supports the OpenAI-compat `/v1/chat/completions`, the native v0 - * `/api/v0/chat/completions`, and any future `/api/vN/chat/completions`. - */ -export function nativeLoadUrl(chatUrl: string): string { - const trimmed = chatUrl.replace(/\/(v1|api\/v\d+)\/chat\/completions\/?$/, ''); - return `${trimmed}/api/v1/models/load`; -} - -export function nativeUnloadUrl(chatUrl: string): string { - const trimmed = chatUrl.replace(/\/(v1|api\/v\d+)\/chat\/completions\/?$/, ''); - return `${trimmed}/api/v1/models/unload`; -} - -/** - * Load a model into memory. Blocks until LM Studio reports `status: "loaded"` - * or the timeout fires. Throws on transport error, non-2xx status, or - * timeout — callers decide whether to swallow or surface. - */ -export async function loadModel( - chatUrl: string, - headers: Record, - model: string, - options: LoadOptions = {}, -): Promise { - const url = nativeLoadUrl(chatUrl); - const body: Record = { model, ...options }; - let resp: Response; - try { - resp = await fetchWithTimeout( - url, - { method: 'POST', headers, body: JSON.stringify(body) }, - LOAD_TIMEOUT_MS, - ); - } catch (err) { - if (err instanceof Error && err.name === 'AbortError') { - throw new Error( - `lmstudio load timed out after ${LOAD_TIMEOUT_MS / 1000}s for model "${model}"`, - ); - } - throw err; - } - if (!resp.ok) { - const text = await resp.text().catch(() => ''); - // Surface common-causes guidance alongside the raw 500 body so the - // operator has a checklist instead of just "Failed to load model.". - // LM Studio's own error message is intentionally terse; the real - // diagnosis is almost always one of these four: - const hint = - resp.status === 404 - ? `(404: native /api/v1/models/load is LM Studio 0.4+ only — older builds will need a manual load in the GUI)` - : `(common causes: model not downloaded in LM Studio; chat template missing or incompatible; insufficient VRAM/RAM; quantization mismatch — open LM Studio's "Developer" panel to see the underlying load error)`; - throw new Error( - `lmstudio load failed (${resp.status}) for model "${model}": ${text || 'no body'} ${hint}`, - ); - } - const result = (await resp.json()) as LoadResult; - logger.info('lmstudio: model loaded', { - model, - instance_id: result.instance_id, - load_time_seconds: result.load_time_seconds, - }); - return result; -} - -/** - * Unload a previously-loaded model instance. - */ -export async function unloadModel( - chatUrl: string, - headers: Record, - instance_id: string, -): Promise { - const url = nativeUnloadUrl(chatUrl); - let resp: Response; - try { - resp = await fetchWithTimeout( - url, - { - method: 'POST', - headers, - body: JSON.stringify({ instance_id }), - }, - UNLOAD_TIMEOUT_MS, - ); - } catch (err) { - if (err instanceof Error && err.name === 'AbortError') { - throw new Error( - `lmstudio unload timed out after ${UNLOAD_TIMEOUT_MS / 1000}s for "${instance_id}"`, - ); - } - throw err; - } - if (!resp.ok) { - const text = await resp.text().catch(() => ''); - throw new Error( - `lmstudio unload failed (${resp.status}): ${text || 'no body'} (instance_id="${instance_id}")`, - ); - } - const result = (await resp.json()) as UnloadResult; - logger.info('lmstudio: model unloaded', { instance_id: result.instance_id }); - return result; -} diff --git a/harness/src/provider-lmstudio/main.ts b/harness/src/provider-lmstudio/main.ts deleted file mode 100644 index 435c29b93..000000000 --- a/harness/src/provider-lmstudio/main.ts +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env node -import { bootstrapWorker } from '../runtime/worker.js'; -import { register } from './register.js'; - -await bootstrapWorker({ - name: 'provider-lmstudio', - description: - 'LM Studio (localhost) Chat Completions streaming provider on the iii bus (provider::lmstudio::stream + ::complete).', - register: (iii, ctx) => register(iii, ctx), -}); diff --git a/harness/src/provider-lmstudio/refresh-fn.ts b/harness/src/provider-lmstudio/refresh-fn.ts deleted file mode 100644 index 85aa5aabd..000000000 --- a/harness/src/provider-lmstudio/refresh-fn.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * `provider::lmstudio::refresh_models` — bus function the UI (or a script) - * can call to re-discover loaded LM Studio models without restarting the - * worker. Wraps `discoverAndRegister`. - * - * Returns `{ registered: string[] }` — the IDs of all models that were - * (re-)written into the catalog on this call. Idempotent: re-registering - * the same model is a no-op state set. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import { buildAuthHeaders } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; - -export const FUNCTION_ID = 'provider::lmstudio::refresh_models'; - -export type RefreshResult = { - registered: string[]; -}; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (): Promise => { - try { - const headers = await buildAuthHeaders(iii, worker.default_api_url); - const registered = await discoverAndRegister(iii, worker.default_api_url, headers); - return { registered }; - } catch (err) { - // Never throw across the bus boundary — refresh is a best-effort - // utility and a failed refresh shouldn't tank the calling UI. - logger.warn('provider::lmstudio::refresh_models failed', { - err: String(err), - }); - return { registered: [] }; - } - }, - { - description: - 'Re-discover loaded LM Studio models and register each into the iii models catalog. Idempotent.', - }, - ); -} diff --git a/harness/src/provider-lmstudio/register.ts b/harness/src/provider-lmstudio/register.ts deleted file mode 100644 index 79dc55c6f..000000000 --- a/harness/src/provider-lmstudio/register.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { loadConfig } from '../runtime/config.js'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import { declareProvider } from '../runtime/provider-resolve.js'; -import { buildAuthHeaders, PROVIDER_ID } from './auth.js'; -import { register as registerComplete } from './complete.js'; -import { loadWorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; -import { register as registerLoad } from './load-fn.js'; -import { register as registerRefresh } from './refresh-fn.js'; -import { register as registerStream } from './stream-fn.js'; -import { register as registerUnload } from './unload-fn.js'; - -export async function register(iii: ISdk, ctx: { configPath: string }): Promise { - const cfg = await loadConfig(ctx.configPath); - const worker = loadWorkerConfig(cfg); - registerComplete(iii, worker); - registerStream(iii, worker); - registerRefresh(iii, worker); - registerLoad(iii, worker); - registerUnload(iii, worker); - - // Self-declare into the harness configuration schema. The api_url is - // env-driven (LMSTUDIO_BASE_URL) so we don't pin it as a stored default; - // only max_tokens is seeded for the form. - void declareProvider(iii, { - id: PROVIDER_ID, - display_name: 'lm studio', - credential_env_var: 'LMSTUDIO_API_KEY', - defaults: { max_tokens: worker.default_max_tokens }, - supports_model_listing: true, - }); - - // Fire-and-forget startup discovery: probe LM Studio's /api/v0/models - // and register each currently-loaded LLM so the picker shows real model - // IDs. Wrapped in setImmediate so a slow/unreachable LM Studio host - // doesn't block the rest of the harness from coming up — the auth and - // models-catalog workers may also still be registering when this runs. - setImmediate(() => { - runStartupDiscovery(iii, worker.default_api_url).catch((err) => { - logger.warn('lmstudio startup discovery threw', { err: String(err) }); - }); - }); -} - -async function runStartupDiscovery(iii: ISdk, chatUrl: string): Promise { - try { - const headers = await buildAuthHeaders(iii, chatUrl); - await discoverAndRegister(iii, chatUrl, headers); - } catch (err) { - // discoverAndRegister already logs its own failures; this catch - // covers the buildAuthHeaders call. - logger.warn('lmstudio startup discovery: header build failed', { - err: String(err), - }); - } -} diff --git a/harness/src/provider-lmstudio/sse.ts b/harness/src/provider-lmstudio/sse.ts deleted file mode 100644 index f635e29ac..000000000 --- a/harness/src/provider-lmstudio/sse.ts +++ /dev/null @@ -1,308 +0,0 @@ -// Kept separate from provider-openai / provider-kimi so LM Studio-specific -// quirks (cold-load timeouts, GGUF-specific finish reasons) can land without -// coupling the providers. - -import type { AssistantMessage } from '../types/agent-message.js'; -import type { ContentBlock } from '../types/content.js'; -import type { AssistantMessageEvent, ErrorKind, StopReason, Usage } from '../types/stream-event.js'; - -type PartialToolCall = { id: string; function_id: string; args_json: string }; - -export type PartialState = { - text: string; - /** - * Accumulated reasoning content from thinking-mode models served by - * LM Studio (qwen3-thinking, glm-thinking, deepseek-r1, etc.). They - * stream reasoning tokens on `delta.reasoning_content` using the same - * convention as Moonshot's Kimi K2. Persisted as a `thinking` - * ContentBlock so subsequent requests can echo it back via - * `reasoning_content` — required by some templates when thinking is - * enabled and the assistant turn carries tool_calls. - */ - reasoning_text: string; - tool_calls: PartialToolCall[]; - usage: Usage; - stop_reason: StopReason; - /** - * Set true the first time we observe a chunk carrying a non-null - * `finish_reason`. Used by stream.ts to distinguish a legitimate - * stream end ("LM Studio said stop") from an abrupt EOF ("connection - * dropped before LM Studio finished"). Without this, every silent - * drop falls through as `stop_reason='end'` and the user sees a - * truncated reply with no error indication. - */ - saw_finish_reason: boolean; -}; - -export function emptyPartial(): PartialState { - return { - text: '', - reasoning_text: '', - tool_calls: [], - usage: { input: 0, output: 0, cache_read: 0, cache_write: 0 }, - stop_reason: 'end', - saw_finish_reason: false, - }; -} - -function buildContent(state: PartialState): ContentBlock[] { - const out: ContentBlock[] = []; - // Thinking goes FIRST so the persisted content order matches what the - // model emitted: think → answer / tool_call. Wire-messages re-emits - // it as `reasoning_content` on the next request. - if (state.reasoning_text.length > 0) { - out.push({ type: 'thinking', text: state.reasoning_text }); - } - if (state.text.length > 0) out.push({ type: 'text', text: state.text }); - for (const tc of state.tool_calls) { - if (tc.function_id.length === 0) continue; - let args: unknown = {}; - if (tc.args_json.length > 0) { - try { - args = JSON.parse(tc.args_json); - } catch { - args = null; - } - } - out.push({ type: 'function_call', id: tc.id, function_id: tc.function_id, arguments: args }); - } - return out; -} - -export function buildPartial( - state: PartialState, - model: string, - provider: string, -): AssistantMessage { - return { - role: 'assistant', - content: buildContent(state), - stop_reason: state.stop_reason, - error_message: null, - error_kind: null, - usage: state.usage, - model, - provider, - timestamp: Date.now(), - }; -} - -export function buildFinal(state: PartialState, model: string, provider: string): AssistantMessage { - return buildPartial(state, model, provider); -} - -export function mapFinishReason(s: string): StopReason { - if (s === 'stop') return 'end'; - if (s === 'length') return 'length'; - if (s === 'tool_calls' || s === 'function_call') return 'function_call'; - return 'end'; -} - -export function mergeUsage(usage: Record, into: Usage): void { - const num = (k: string) => (typeof usage[k] === 'number' ? (usage[k] as number) : 0); - into.input = (into.input ?? 0) + num('prompt_tokens') + num('input_tokens'); - into.output = (into.output ?? 0) + num('completion_tokens') + num('output_tokens'); - for (const parent of ['prompt_tokens_details', 'input_tokens_details']) { - const d = usage[parent] as Record | undefined; - if (d && typeof d.cached_tokens === 'number') { - into.cache_read = (into.cache_read ?? 0) + d.cached_tokens; - } - } -} - -/** - * Patterns that indicate an LM Studio model failed to load (either - * because the user never loaded it, JIT-load crashed, or the GGUF - * couldn't initialize). Centralized so the classifier and the - * error-message formatter agree on what counts as a load failure, and - * so the auto-load retry in `stream.ts` can key off the same set. - */ -const LOAD_FAILURE_PATTERN = - /no model (is )?loaded|model not found|please load|model has crashed|model_load_failed|failed to load (?:llm|model)|exit code/i; - -export function isLoadFailureMessage(message: string): boolean { - return LOAD_FAILURE_PATTERN.test(message); -} - -export function classifyLmstudioError(message: string, status?: number): ErrorKind { - // Localhost LM Studio rarely returns 401/403, but corporate proxies and - // authenticated deployments can — keep the mapping so downstream retry - // logic does the right thing. - if (status === 401 || status === 403) return 'auth_expired'; - if (status === 429) return 'rate_limited'; - if (status && status >= 500) return 'transient'; - // Load failures cover everything the user can fix by loading the - // right model: "no model loaded", "model not found", "please load", - // and the post-JIT crash shapes ("The model has crashed", "Failed to - // load LLM …", "Exit code: null"). All map to transient so the - // orchestrator retries and the UI doesn't show a hard dead-end. - if (isLoadFailureMessage(message)) return 'transient'; - if (/context length|too many tokens/i.test(message)) return 'context_overflow'; - return 'permanent'; -} - -/** - * Prepend a user-actionable hint when the message is a load-failure - * shape. Keeps the original wire text intact (after the hint) so users - * and logs still see exactly what LM Studio said. - */ -export function formatLmstudioError(message: string): string { - if (!isLoadFailureMessage(message)) return message; - return ( - `LM Studio could not load the model. Try loading it first via ` + - `\`provider::lmstudio::load_model\`, or pick a different model in the picker. ` + - `Original error: ${message}` - ); -} - -export function syntheticErrorEvent( - message: string, - model: string, - provider: string, - error_kind: ErrorKind = 'transient', -): AssistantMessageEvent { - const formatted = formatLmstudioError(message); - // Carry the formatted text ONLY in `error_message` (which the UI's - // translate layer routes through the `stop-reason` notice channel). - // Do NOT inject it as a `text` ContentBlock — pre-fix that meant a - // malicious LM Studio backend or MITM could inject "tool approved" - // lines or attacker-controlled markup into the assistant message - // stream, where it would be persisted and re-fed to the next - // provider call as trusted context (a prompt-injection vector via - // the local model server's error channel). - const final: AssistantMessage = { - role: 'assistant', - content: [], - stop_reason: 'error', - error_message: formatted, - error_kind, - usage: null, - model, - provider, - timestamp: Date.now(), - }; - return { type: 'error', error: final }; -} - -/** - * Extract a human-readable error message from an SSE error chunk. LM - * Studio (and other OpenAI-compatible servers) send a JSON chunk shaped - * `{"error": {"message": "...", ...}}` or `{"error": "..."}` when prompt - * rendering or generation fails mid-stream after the HTTP 200 was - * already committed. Returns null when the chunk has no error. - */ -export function extractErrorMessage(chunk: Record): string | null { - const err = chunk.error; - if (!err) return null; - if (typeof err === 'string' && err.length > 0) return err; - if (typeof err === 'object') { - const obj = err as Record; - if (typeof obj.message === 'string' && obj.message.length > 0) return obj.message; - // Some servers nest the message under .error.error_message or .error.detail - if (typeof obj.error_message === 'string' && obj.error_message.length > 0) { - return obj.error_message; - } - if (typeof obj.detail === 'string' && obj.detail.length > 0) return obj.detail; - } - return null; -} - -export function handleChunk( - chunk: Record, - state: PartialState, - model: string, - provider: string, -): AssistantMessageEvent[] { - const events: AssistantMessageEvent[] = []; - - // SSE error chunks: LM Studio surfaces template-render errors, OOM, - // and other mid-stream failures as a JSON chunk with an `error` field - // and no `choices`. Without this branch we'd ignore the chunk (no - // choices) and later fall through with a generic "stream closed - // mid-response" message — losing the specific server-side reason. - const errMsg = extractErrorMessage(chunk); - if (errMsg) { - state.stop_reason = 'error'; - state.saw_finish_reason = true; // prevent the EOF-without-finish guard - return [syntheticErrorEvent(errMsg, model, provider, classifyLmstudioError(errMsg))]; - } - - const usage = chunk.usage as Record | undefined; - if (usage) mergeUsage(usage, state.usage); - const choices = chunk.choices; - if (!Array.isArray(choices) || choices.length === 0) return events; - const choice = choices[0] as Record; - const finish = typeof choice.finish_reason === 'string' ? choice.finish_reason : null; - if (finish) { - state.stop_reason = mapFinishReason(finish); - state.saw_finish_reason = true; - } - const delta = choice.delta as Record | undefined; - if (!delta) return events; - - // Reasoning tokens — thinking-mode models served by LM Studio (qwen3, - // glm, deepseek-r1, …) stream these on `delta.reasoning_content` - // BEFORE any content/tool_calls. Surface as thinking_* events and - // persist so the round-trip carries them back via `reasoning_content`. - if (typeof delta.reasoning_content === 'string' && delta.reasoning_content.length > 0) { - if (state.reasoning_text.length === 0) { - events.push({ type: 'thinking_start', partial: buildPartial(state, model, provider) }); - } - state.reasoning_text += delta.reasoning_content; - events.push({ - type: 'thinking_delta', - partial: buildPartial(state, model, provider), - delta: delta.reasoning_content, - }); - } - - if (typeof delta.content === 'string' && delta.content.length > 0) { - if (state.text.length === 0) { - events.push({ type: 'text_start', partial: buildPartial(state, model, provider) }); - } - state.text += delta.content; - events.push({ - type: 'text_delta', - partial: buildPartial(state, model, provider), - delta: delta.content, - }); - } - - const tool_calls = delta.tool_calls; - if (Array.isArray(tool_calls)) { - for (const tc of tool_calls) { - if (!tc || typeof tc !== 'object') continue; - const tcObj = tc as Record; - const rawIndex = typeof tcObj.index === 'number' ? tcObj.index : 0; - // Reject attacker-controlled indices that would force unbounded - // allocation. Without this guard, a hostile (or buggy) LM Studio - // backend can DoS the worker by sending `{"index": 1e9}` in a - // delta.tool_calls entry — the while-loop below would allocate - // billions of slots. 256 is well above any realistic tool-call - // fan-out from a single model turn. - if (!Number.isInteger(rawIndex) || rawIndex < 0 || rawIndex > 256) { - continue; - } - const index = rawIndex; - while (state.tool_calls.length <= index) { - state.tool_calls.push({ id: '', function_id: '', args_json: '' }); - } - const entry = state.tool_calls[index]; - if (!entry) continue; - if (typeof tcObj.id === 'string' && tcObj.id.length > 0) entry.id = tcObj.id; - const fn = tcObj.function as Record | undefined; - if (fn) { - if (typeof fn.name === 'string' && fn.name.length > 0) entry.function_id = fn.name; - if (typeof fn.arguments === 'string') { - entry.args_json += fn.arguments; - events.push({ - type: 'functioncall_delta', - partial: buildPartial(state, model, provider), - delta: fn.arguments, - }); - } - } - } - } - return events; -} diff --git a/harness/src/provider-lmstudio/stream-fn.ts b/harness/src/provider-lmstudio/stream-fn.ts deleted file mode 100644 index 51770d6fa..000000000 --- a/harness/src/provider-lmstudio/stream-fn.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { ChannelWriter } from 'iii-sdk'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import { - ProviderStreamInputJsonSchema, - ProviderStreamOutputJsonSchema, - ProviderStreamRuntimeInputSchema, -} from '../types/provider.js'; -import { isTerminal } from '../types/stream-event.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { streamLmstudio } from './stream.js'; - -export const FUNCTION_ID = 'provider::lmstudio::stream'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (raw: unknown) => { - const input = ProviderStreamRuntimeInputSchema.parse(raw); - // The iii-sdk auto-hydrates `writer_ref` (a StreamChannelRef on the - // wire) into a `ChannelWriter` instance before this handler runs. - const writer = input.writer_ref as ChannelWriter; - const cfg = await buildConfig(iii, worker, input.model); - try { - const events = streamLmstudio({ - cfg, - system_prompt: input.system_prompt ?? '', - messages: input.messages as AgentMessage[], - tools: input.tools as import('../types/function.js').AgentFunction[], - }); - for await (const ev of events) { - writer.sendMessage(JSON.stringify(ev)); - if (isTerminal(ev)) break; - } - } catch (err) { - logger.warn('provider::lmstudio::stream failed mid-flight', { err: String(err) }); - } finally { - try { - writer.close(); - } catch (err) { - logger.debug('writer.close failed', { err: String(err) }); - } - } - return { ok: true }; - }, - { - description: - 'Stream a single assistant turn from a local LM Studio Chat Completions server into the caller-supplied channel.', - request_format: ProviderStreamInputJsonSchema as Record, - response_format: ProviderStreamOutputJsonSchema as Record, - }, - ); -} diff --git a/harness/src/provider-lmstudio/stream.ts b/harness/src/provider-lmstudio/stream.ts deleted file mode 100644 index 7f4cc6947..000000000 --- a/harness/src/provider-lmstudio/stream.ts +++ /dev/null @@ -1,413 +0,0 @@ -// Kept separate from provider-openai / provider-kimi so LM Studio-specific -// request options (e.g. keep_alive) can be added without coupling providers. - -import { logger } from '../runtime/otel.js'; -import type { AgentMessage, AssistantMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import type { AssistantMessageEvent } from '../types/stream-event.js'; -import { loadModel } from './load.js'; -import { - buildFinal, - classifyLmstudioError, - emptyPartial, - handleChunk, - isLoadFailureMessage, - syntheticErrorEvent, -} from './sse.js'; -import type { ChatCompletionsConfig } from './types.js'; -import { toOpenaiMessages } from './wire-messages.js'; -import { functionsToOpenai } from './wire-tools.js'; - -/** Catalog placeholder id; routes to "whatever LM Studio currently has loaded". */ -export const PLACEHOLDER_MODEL_ID = 'lmstudio-local'; - -/** Connect + first-byte timeout for any LM Studio HTTP call. */ -const FETCH_TIMEOUT_MS = 30_000; - -export type StreamArgs = { - cfg: ChatCompletionsConfig; - system_prompt: string; - messages: AgentMessage[]; - tools: AgentFunction[]; -}; - -/** - * Run `fetch` with an AbortController-based timeout. Without this a - * misconfigured URL (typo, dead LAN host) hangs for ~75s (macOS SYN - * timeout) and the UI stays stuck on "thinking…" the whole time. - */ -async function fetchWithTimeout( - url: string, - init: RequestInit, - timeoutMs: number, -): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), timeoutMs); - try { - return await fetch(url, { ...init, signal: controller.signal }); - } finally { - clearTimeout(timer); - } -} - -/** - * Strip C0 control chars (except TAB / LF / CR) and DEL. Used to - * sanitize non-2xx response bodies before they land in user-visible - * messages or log lines. Written as a charCodeAt loop rather than - * `/[\x00-\x1f]/` so Biome's `noControlCharactersInRegex` is happy. - */ -function stripControlChars(s: string): string { - let out = ''; - for (let i = 0; i < s.length; i++) { - const code = s.charCodeAt(i); - if (code < 32 && code !== 9 && code !== 10 && code !== 13) continue; - if (code === 127) continue; - out += s[i]; - } - return out; -} - -/** - * Build the chat-completions URL's sibling `/v1/models` endpoint by - * trimming the trailing `/chat/completions` (if present). Works for the - * default `…/v1/chat/completions` and for any user-supplied URL that - * follows the same pattern. - */ -function modelsUrl(chatUrl: string): string { - const trimmed = chatUrl.replace(/\/chat\/completions\/?$/, ''); - return `${trimmed}/models`; -} - -/** - * When the request comes in with the catalog placeholder `lmstudio-local`, - * ask LM Studio which model is actually loaded and substitute that id. If - * discovery fails (network error, empty list, malformed response), return - * the placeholder unchanged — the downstream fetch will then surface a - * clear error rather than this helper silently masking one. - * - * For explicit model ids we forward verbatim. If the model isn't currently - * loaded, either LM Studio's JIT setting auto-loads it, or the caller can - * pre-load it via the `provider::lmstudio::load_model` bus function. - */ -async function resolveModel( - cfg: ChatCompletionsConfig, - headers: Record, -): Promise { - if (cfg.model !== PLACEHOLDER_MODEL_ID) return cfg.model; - try { - const resp = await fetchWithTimeout(modelsUrl(cfg.url), { method: 'GET', headers }, 5_000); - if (!resp.ok) { - logger.warn('lmstudio /v1/models returned non-2xx; using placeholder', { - status: resp.status, - }); - return cfg.model; - } - const parsed = (await resp.json()) as { data?: Array<{ id?: string }> }; - const first = parsed.data?.find( - (m) => typeof m?.id === 'string' && (m.id as string).length > 0, - )?.id; - if (first) { - logger.info('lmstudio: resolved placeholder to loaded model', { model: first }); - return first; - } - logger.warn('lmstudio /v1/models returned empty data; using placeholder', {}); - return cfg.model; - } catch (err) { - logger.warn('lmstudio /v1/models discovery failed; using placeholder', { err: String(err) }); - return cfg.model; - } -} - -/** - * One attempt at the chat-completions request + SSE drain. Yields the - * normal stream events. Pulled out of `streamLmstudio` so the outer - * generator can call it twice when the first attempt fails with a - * load-failure shape — auto-loading the model in between. - */ -async function* attemptStream( - cfg: ChatCompletionsConfig, - headers: Record, - effectiveModel: string, - system_prompt: string, - messages: AgentMessage[], - tools: AgentFunction[], -): AsyncGenerator { - const body: Record = { - model: effectiveModel, - max_completion_tokens: cfg.max_tokens, - messages: toOpenaiMessages(messages, system_prompt), - stream: true, - stream_options: { include_usage: true }, - }; - if (tools.length > 0) body.tools = functionsToOpenai(tools); - - let resp: Response; - try { - resp = await fetchWithTimeout( - cfg.url, - { method: 'POST', headers, body: JSON.stringify(body) }, - FETCH_TIMEOUT_MS, - ); - } catch (err) { - const msg = - err instanceof Error && err.name === 'AbortError' - ? `lmstudio fetch timed out after ${FETCH_TIMEOUT_MS / 1000}s (is ${cfg.url} reachable?)` - : `lmstudio fetch failed: ${String(err)}`; - yield syntheticErrorEvent(msg, effectiveModel, cfg.provider_name); - return; - } - if (!resp.ok) { - const text = await resp.text().catch(() => ''); - // Truncate + strip control chars before surfacing — non-2xx - // responses can carry HTML auth-redirect bodies, proxy error - // pages, or attacker-controlled content. Length cap keeps the - // message readable; control-char strip avoids ANSI / newline - // injection into log lines and live regions. Uses a charCodeAt - // loop rather than a regex literal so Biome's - // `noControlCharactersInRegex` doesn't trip. - const safeText = stripControlChars(text).slice(0, 256); - yield syntheticErrorEvent( - safeText || `lmstudio http ${resp.status}`, - cfg.model, - cfg.provider_name, - classifyLmstudioError(safeText, resp.status), - ); - return; - } - const partial: AssistantMessage = { - role: 'assistant', - content: [], - stop_reason: 'end', - error_message: null, - error_kind: null, - usage: null, - model: effectiveModel, - provider: cfg.provider_name, - timestamp: Date.now(), - }; - yield { type: 'start', partial }; - - const state = emptyPartial(); - if (!resp.body) { - yield syntheticErrorEvent('lmstudio response missing body', effectiveModel, cfg.provider_name); - return; - } - const reader = resp.body.getReader(); - const decoder = new TextDecoder(); - let buf = ''; - // Counts successfully-parsed SSE data chunks. If we reach EOF with zero, - // LM Studio returned a 200 with a non-SSE body (e.g. an HTML dashboard - // page when no model is loaded on some builds) — surface that as an - // explicit error rather than silently emitting an empty `done`. - let parsedChunkCount = 0; - try { - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - let idx = buf.indexOf('\n\n'); - while (idx >= 0) { - const block = buf.slice(0, idx); - buf = buf.slice(idx + 2); - const dataLine = parseDataLine(block); - idx = buf.indexOf('\n\n'); - if (dataLine === null) continue; - if (dataLine === '[DONE]') { - yield { type: 'done', message: buildFinal(state, effectiveModel, cfg.provider_name) }; - return; - } - let parsed: Record | null = null; - try { - parsed = JSON.parse(dataLine) as Record; - } catch { - continue; - } - if (parsed) { - parsedChunkCount++; - for (const e of handleChunk(parsed, state, effectiveModel, cfg.provider_name)) { - yield e; - // handleChunk returns an `error` event when LM Studio sent an - // SSE error chunk (template render failure, mid-stream OOM, - // model unloaded). Stop reading immediately so the rest of - // the body (typically just a connection close) doesn't fall - // into the EOF-without-finish_reason guard and overwrite the - // server's specific error message. - if (e.type === 'error') return; - } - } - } - } - } catch (err) { - logger.warn('lmstudio stream read failed', { err: String(err) }); - yield syntheticErrorEvent( - `stream read failed: ${String(err)}`, - effectiveModel, - cfg.provider_name, - ); - return; - } - if (parsedChunkCount === 0) { - // Phrased to avoid matching `isLoadFailureMessage` — this guard - // usually means a wrong URL (LM Studio's HTML dashboard) and an - // auto-load retry would not help. We keep the diagnostic text but - // describe the cause as "non-SSE body" rather than mentioning - // "no model is loaded". - yield syntheticErrorEvent( - 'lmstudio returned a 200 response with a non-SSE body (no parseable chunks). The endpoint URL is likely wrong, or LM Studio served its dashboard page in place of an SSE stream.', - cfg.model, - cfg.provider_name, - ); - return; - } - // We parsed some chunks but never saw `data: [DONE]` AND never saw a - // `finish_reason` in any chunk. That means LM Studio closed the body - // mid-response — typical causes: GPU OOM during generation, model - // unloaded mid-stream, host connection dropped, or context exhausted - // by the running output. Without this guard the user sees a silently - // truncated reply with no error, because state.stop_reason is still - // its default 'end'. Promote to an explicit error so the UI can show - // "stream closed mid-response" instead of pretending success. - if (!state.saw_finish_reason) { - const partial = buildFinal(state, effectiveModel, cfg.provider_name); - const tokenHint = - partial.usage && (partial.usage.output ?? 0) > 0 - ? ` after ~${partial.usage.output} output tokens` - : ''; - yield syntheticErrorEvent( - `lmstudio stream closed mid-response${tokenHint} — the server ended the SSE body without a [DONE] marker or finish_reason. Common causes: model unloaded, GPU OOM, host disconnect, or context exhausted during generation.`, - effectiveModel, - cfg.provider_name, - 'transient', - ); - return; - } - yield { type: 'done', message: buildFinal(state, effectiveModel, cfg.provider_name) }; -} - -export async function* streamLmstudio({ - cfg, - system_prompt, - messages, - tools, -}: StreamArgs): AsyncGenerator { - const authName = cfg.auth_header_name ?? 'Authorization'; - const authPrefix = cfg.auth_value_prefix ?? 'Bearer '; - const headers: Record = { - 'content-type': 'application/json', - [authName]: `${authPrefix}${cfg.api_key}`, - }; - for (const [k, v] of cfg.extra_headers ?? []) headers[k] = v; - - // Resolve `lmstudio-local` → the actual loaded model id, so the catalog - // placeholder "just works" without the caller having to know what's loaded. - const effectiveModel = await resolveModel(cfg, headers); - - // Auto-load retry: on the first attempt, if the response is a - // load-failure error AND we have not yet streamed any real progress - // (text, tool, or thinking deltas) to the user, swallow the failure, - // call `provider::lmstudio::load_model` for the resolved model id, - // then retry the chat completion exactly once. This converts the - // common "user picked a model that's listed but not actually loaded" - // failure into a working call. The retry guard ensures we never loop: - // if the second attempt fails too, the error reaches the caller. - let retried = false; - for (;;) { - // Buffer the first `start` event so we can DROP it (rather than - // surface a confusing "started, then errored" pair) when we decide - // to retry. Any non-start event flips `committed` to true and we - // stream straight through from that point on. - let bufferedStart: AssistantMessageEvent | null = null; - let committed = false; - let retryRequested = false; - - for await (const ev of attemptStream( - cfg, - headers, - effectiveModel, - system_prompt, - messages, - tools, - )) { - if (!committed) { - if (ev.type === 'start') { - bufferedStart = ev; - continue; - } - if (ev.type === 'error' && !retried && isLoadFailureMessage(ev.error.error_message ?? '')) { - retryRequested = true; - break; - } - // First non-start, non-retriable event — flush the buffered start - // so the downstream sees the canonical `start → … → done|error` - // sequence. - if (bufferedStart) { - yield bufferedStart; - bufferedStart = null; - } - committed = true; - } - yield ev; - } - - if (retryRequested) { - logger.info('lmstudio: auto-load retry triggered by load-failure error', { - model: effectiveModel, - }); - try { - await loadModel(cfg.url, headers, effectiveModel); - } catch (loadErr) { - // The auto-load itself failed (404 on older LM Studio without the - // native v1 endpoints, model that won't load on this hardware, - // etc.). Surface as transient so the operator/UI knows it can - // potentially be retried after fixing the environment. - yield syntheticErrorEvent( - `auto-load failed for "${effectiveModel}": ${String(loadErr)}`, - effectiveModel, - cfg.provider_name, - 'transient', - ); - return; - } - retried = true; - continue; - } - - // Inner finished naturally (committed events + done/error already - // forwarded). If by some path it only emitted `start`, flush it so - // we never leave the downstream wondering. - if (bufferedStart && !committed) yield bufferedStart; - return; - } -} - -function parseDataLine(block: string): string | null { - let data: string | null = null; - for (const line of block.split('\n')) { - if (line.startsWith('data: ')) data = line.slice('data: '.length); - } - return data; -} - -export async function collect( - events: AsyncIterable, -): Promise { - let last: AssistantMessage | null = null; - for await (const ev of events) { - if (ev.type === 'done') return ev.message; - if (ev.type === 'error') return ev.error; - if ('partial' in ev) last = ev.partial; - } - return ( - last ?? { - role: 'assistant', - content: [], - stop_reason: 'error', - error_message: 'stream closed without final', - error_kind: 'transient', - usage: null, - model: 'lmstudio', - provider: 'lmstudio', - timestamp: Date.now(), - } - ); -} diff --git a/harness/src/provider-lmstudio/types.ts b/harness/src/provider-lmstudio/types.ts deleted file mode 100644 index 6884b3531..000000000 --- a/harness/src/provider-lmstudio/types.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { Credential } from '../runtime/provider-resolve.js'; - -export type ChatCompletionsConfig = { - url: string; - provider_name: string; - model: string; - api_key: string; - /** Defaults to "Authorization". */ - auth_header_name?: string; - /** Defaults to "Bearer ". */ - auth_value_prefix?: string; - extra_headers?: Array; - max_tokens: number; -}; - -export function configFromCredential( - url: string, - provider_name: string, - model: string, - cred: Credential, - max_tokens: number, -): ChatCompletionsConfig { - const api_key = cred.type === 'api_key' ? cred.key : cred.access_token; - return { url, provider_name, model, api_key, max_tokens }; -} diff --git a/harness/src/provider-lmstudio/unload-fn.ts b/harness/src/provider-lmstudio/unload-fn.ts deleted file mode 100644 index 0b6730ec9..000000000 --- a/harness/src/provider-lmstudio/unload-fn.ts +++ /dev/null @@ -1,47 +0,0 @@ -/** - * `provider::lmstudio::unload_model` — bus function to release a loaded - * LM Studio model instance. Wraps `POST /api/v1/models/unload`. - * - * Pairs with `provider::lmstudio::load_model`. Useful for freeing memory - * before loading a different model on RAM-constrained hosts, or as part - * of an explicit teardown. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import { buildAuthHeaders } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { type UnloadResult, unloadModel } from './load.js'; - -export const FUNCTION_ID = 'provider::lmstudio::unload_model'; - -export type UnloadModelPayload = { - instance_id: string; -}; - -export type UnloadModelResult = UnloadResult | { ok: false; error: string }; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (raw: unknown): Promise => { - const payload = raw as Partial | null | undefined; - if (!payload || typeof payload.instance_id !== 'string' || payload.instance_id.length === 0) { - return { ok: false, error: 'invalid payload: { instance_id: string } is required' }; - } - try { - const headers = await buildAuthHeaders(iii, worker.default_api_url); - return await unloadModel(worker.default_api_url, headers, payload.instance_id); - } catch (err) { - logger.warn('provider::lmstudio::unload_model failed', { - instance_id: payload.instance_id, - err: String(err), - }); - return { ok: false, error: String(err) }; - } - }, - { - description: 'Unload an LM Studio model instance via POST /api/v1/models/unload.', - }, - ); -} diff --git a/harness/src/provider-lmstudio/wire-messages.ts b/harness/src/provider-lmstudio/wire-messages.ts deleted file mode 100644 index be52a9542..000000000 --- a/harness/src/provider-lmstudio/wire-messages.ts +++ /dev/null @@ -1,120 +0,0 @@ -// LM Studio speaks OpenAI's Chat Completions wire format. Kept separate from -// provider-openai / provider-kimi so any LM Studio-specific extensions (e.g. -// `keep_alive`, vision parts for multimodal GGUFs) can land without coupling -// the providers. - -import { logger } from '../runtime/otel.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import { formatFunctionResultContent } from '../types/wire.js'; - -/** - * Some local model templates (notably qwen3's jinja) require at least one - * `role: 'user'` message and abort with "No user query found in messages" - * otherwise. That condition is reachable in normal agentic operation: - * after a tool call cycle whose ancestors got summarised away by async - * compaction, the flat-state can be [assistant, tool, assistant, tool, …] - * with the original user message gone. Without this safety net we'd send - * the request, LM Studio would error mid-stream, and the user would see - * a cryptic "stream closed mid-response" instead of a clean continuation. - * - * The placeholder is intentionally minimal ("(continue)") so the model - * understands it as a continuation directive — matches what LiteLLM / - * OpenRouter do for the same template constraint. - */ -export const PLACEHOLDER_USER_MESSAGE = '(continue)'; - -function hasUserMessage(wire: readonly unknown[]): boolean { - for (const m of wire) { - if (m && typeof m === 'object' && (m as { role?: unknown }).role === 'user') { - return true; - } - } - return false; -} - -export function toOpenaiMessages(messages: AgentMessage[], system_prompt: string): unknown[] { - const out: unknown[] = []; - // Index of `tool_call_id → out[]` slot for O(1) latest-wins dedup - // of function_result rows. Pre-fix this was an `out.findIndex` scan - // per function_result, which gave O(M²) translation cost when the - // history carried many tool calls — amplified on LM Studio because - // the auto-load retry can re-translate the same history twice in - // one user turn. - const toolResultIndexById = new Map(); - if (system_prompt.length > 0) { - out.push({ role: 'system', content: system_prompt }); - } - for (const m of messages) { - if (m.role === 'user') { - const text = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'text' }> => c.type === 'text', - ) - .map((c) => c.text) - .join('\n'); - out.push({ role: 'user', content: text }); - } else if (m.role === 'assistant') { - const text = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'text' }> => c.type === 'text', - ) - .map((c) => c.text) - .join('\n'); - // Thinking-mode models served by LM Studio (qwen3, glm, deepseek-r1) - // require `reasoning_content` echoed back on assistant tool-call - // messages — same constraint as Kimi K2. sse.ts captures the - // stream into a ThinkingContent block; project it back here. - const reasoning = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'thinking' }> => - c.type === 'thinking', - ) - .map((c) => c.text) - .join(''); - const tool_calls = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'function_call' }> => - c.type === 'function_call', - ) - .map((c) => ({ - id: c.id, - type: 'function', - function: { name: c.function_id, arguments: JSON.stringify(c.arguments) }, - })); - const entry: Record = { role: 'assistant' }; - if (reasoning.length > 0) entry.reasoning_content = reasoning; - if (text.length > 0) entry.content = text; - if (tool_calls.length > 0) entry.tool_calls = tool_calls; - out.push(entry); - } else if (m.role === 'function_result') { - const text = formatFunctionResultContent(m); - const row: Record = { - role: 'tool', - tool_call_id: m.function_call_id, - content: text, - }; - if (m.is_error) row.is_error = true; - // Boundary dedup — see provider-anthropic/wire-messages.ts for why. - // LM Studio's jinja templates vary by GGUF; some accept duplicates, - // some don't. Latest-wins replace keeps every template happy. - // O(1) lookup via the index map (see top-of-function note on - // why we don't scan `out` linearly here). - const existingIdx = toolResultIndexById.get(m.function_call_id); - if (existingIdx !== undefined) { - out[existingIdx] = row; - } else { - toolResultIndexById.set(m.function_call_id, out.length); - out.push(row); - } - } - // custom messages are skipped - } - if (!hasUserMessage(out)) { - logger.warn( - 'lmstudio: no user-role message in request; injecting placeholder to satisfy strict jinja templates (e.g. qwen3 "No user query found")', - { messageCount: out.length }, - ); - out.push({ role: 'user', content: PLACEHOLDER_USER_MESSAGE }); - } - return out; -} diff --git a/harness/src/provider-lmstudio/wire-tools.ts b/harness/src/provider-lmstudio/wire-tools.ts deleted file mode 100644 index c133e4f33..000000000 --- a/harness/src/provider-lmstudio/wire-tools.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { AgentFunction } from '../types/function.js'; - -export function functionsToOpenai(functions: AgentFunction[]): unknown[] { - return functions.map((t) => ({ - type: 'function', - function: { - name: t.name, - description: t.description, - parameters: t.parameters, - }, - })); -} diff --git a/harness/src/provider-openai/auth.ts b/harness/src/provider-openai/auth.ts deleted file mode 100644 index f9a2ae217..000000000 --- a/harness/src/provider-openai/auth.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Resolve the OpenAI credential + runtime settings from the harness provider - * registry (`harness::provider::resolve`) and build a ChatCompletionsConfig. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { clampOutputTokens, getCatalogModel } from '../runtime/output-tokens.js'; -import { resolveProvider } from '../runtime/provider-resolve.js'; -import type { WorkerConfig } from './config.js'; -import { type ChatCompletionsConfig, configFromCredential } from './types.js'; - -export const PROVIDER_ID = 'openai'; - -export async function buildConfig( - iii: ISdk, - worker: WorkerConfig, - model: string, -): Promise { - const resolved = await resolveProvider(iii, PROVIDER_ID); - if (!resolved.credential) { - throw new Error( - 'harness::provider::resolve returned no credential for provider `openai` ' + - '(set an api key in the harness configuration or OPENAI_API_KEY)', - ); - } - const apiUrl = resolved.api_url ?? worker.default_api_url; - const catalog = await getCatalogModel(iii, PROVIDER_ID, model); - const maxTokens = clampOutputTokens({ - modelMaxOutput: catalog?.max_output_tokens, - userOverride: resolved.max_tokens, - workerDefault: worker.default_max_tokens, - }); - const cfg = configFromCredential(apiUrl, PROVIDER_ID, model, resolved.credential, maxTokens); - return catalog ? { ...cfg, catalog } : cfg; -} diff --git a/harness/src/provider-openai/complete.ts b/harness/src/provider-openai/complete.ts deleted file mode 100644 index d72eaeaa9..000000000 --- a/harness/src/provider-openai/complete.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { requireString } from '../runtime/handler.js'; -import type { ISdk } from '../runtime/iii.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { collect, streamOpenai } from './stream.js'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - 'provider::openai::complete', - async (payload: unknown) => { - const obj = (payload ?? {}) as Record; - const model = requireString(obj, 'model'); - const system_prompt = typeof obj.system_prompt === 'string' ? obj.system_prompt : ''; - const messages = Array.isArray(obj.messages) ? (obj.messages as AgentMessage[]) : []; - const tools = Array.isArray(obj.tools) ? (obj.tools as AgentFunction[]) : []; - const cfg = await buildConfig(iii, worker, model); - return await collect(streamOpenai({ cfg, system_prompt, messages, tools })); - }, - { - description: - 'Legacy: drain a streamed OpenAI chat-completion and return the final AssistantMessage.', - }, - ); -} diff --git a/harness/src/provider-openai/config.ts b/harness/src/provider-openai/config.ts deleted file mode 100644 index e2afd4f67..000000000 --- a/harness/src/provider-openai/config.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { getNumber, getSection, getString } from '../runtime/config.js'; - -export type WorkerConfig = { - default_max_tokens: number; - default_api_url: string; -}; - -export const DEFAULT_API_URL = 'https://api.openai.com/v1/chat/completions'; - -export function loadWorkerConfig(cfg: Record): WorkerConfig { - const section = getSection(cfg, 'provider_openai'); - return { - default_max_tokens: getNumber(section, 'default_max_tokens', 8192), - default_api_url: getString(section, 'default_api_url', DEFAULT_API_URL), - }; -} diff --git a/harness/src/provider-openai/discover.ts b/harness/src/provider-openai/discover.ts deleted file mode 100644 index f4859e2b9..000000000 --- a/harness/src/provider-openai/discover.ts +++ /dev/null @@ -1,83 +0,0 @@ -/** - * OpenAI model discovery — hits `GET /v1/models` and registers the - * chat-capable subset into the iii models catalog (cached for the picker). - * The OpenAI list includes embeddings/audio/image models; we filter to - * chat/reasoning families and register them with a default context window. - * - * Best-effort: a missing credential or any upstream error yields `[]`. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { - deriveModelsUrl, - enrichModel, - fetchModelsForDiscovery, - type ModelStub, - reconcileModels, -} from '../runtime/models-discovery.js'; -import { getModelsDevIndex, lookupModelsDev } from '../runtime/modelsdev.js'; -import { logger } from '../runtime/otel.js'; -import { resolveProvider } from '../runtime/provider-resolve.js'; -import { PROVIDER_ID } from './auth.js'; -import type { WorkerConfig } from './config.js'; - -const DEFAULT_CONTEXT_WINDOW = 128_000; - -// Chat/reasoning families we route to. Excludes embeddings, audio, image, -// moderation, realtime, and legacy completion-only model ids. -const CHAT_ID = /^(gpt-|o\d|chatgpt)/i; -const NON_CHAT = - /(embedding|whisper|tts|audio|dall-e|image|moderation|realtime|transcribe|search|babbage|davinci|ada|curie)/i; - -function parseStubs(json: unknown): ModelStub[] { - const data = (json as { data?: unknown })?.data; - if (!Array.isArray(data)) return []; - const out: ModelStub[] = []; - for (const raw of data as Array<{ id?: unknown }>) { - const id = typeof raw.id === 'string' && raw.id.length > 0 ? raw.id : null; - if (!id) continue; - if (!CHAT_ID.test(id) || NON_CHAT.test(id)) continue; - out.push({ id }); - } - return out; -} - -export async function discoverAndRegister(iii: ISdk, worker: WorkerConfig): Promise { - const resolved = await resolveProvider(iii, PROVIDER_ID).catch(() => null); - const cred = resolved?.credential ?? null; - if (!cred) { - // No credential: drop any models a previous run registered so the picker - // reflects the removal instead of showing stale, unusable rows. - logger.info('openai discovery: no credential; pruning catalog', {}); - await reconcileModels(iii, PROVIDER_ID, []); - return []; - } - const key = cred.type === 'api_key' ? cred.key : cred.access_token; - const url = deriveModelsUrl(resolved?.api_url ?? worker.default_api_url); - const fetchResult = await fetchModelsForDiscovery(url, { Authorization: `Bearer ${key}` }); - if (fetchResult.kind === 'auth_error') { - logger.info('openai discovery: invalid credential; pruning catalog', { - status: fetchResult.status, - }); - await reconcileModels(iii, PROVIDER_ID, []); - return []; - } - if (fetchResult.kind !== 'ok') return []; - - const modelsDev = await getModelsDevIndex(); - const models = parseStubs(fetchResult.json).map((stub) => - enrichModel({ - provider: PROVIDER_ID, - api: 'openai-responses', - stub, - defaultContextWindow: DEFAULT_CONTEXT_WINDOW, - modelsDev: lookupModelsDev(modelsDev, PROVIDER_ID, stub.id), - }), - ); - const registered = await reconcileModels(iii, PROVIDER_ID, models); - logger.info('openai discovery: reconciled models', { - count: registered.length, - discovered: models.length, - }); - return registered; -} diff --git a/harness/src/provider-openai/iii.worker.yaml b/harness/src/provider-openai/iii.worker.yaml deleted file mode 100644 index 51e9c21fb..000000000 --- a/harness/src/provider-openai/iii.worker.yaml +++ /dev/null @@ -1,17 +0,0 @@ -iii: v1 -name: provider-openai -language: node -deploy: binary -manifest: package.json -bin: iii-provider-openai -description: OpenAI Chat Completions streaming provider; exposes provider::openai::stream and provider::openai::complete on the iii bus. - -runtime: - kind: node - -scripts: - install: pnpm install - start: node ./dist/provider-openai/main.js --config ./config.yaml - -dependencies: - configuration: "^0.11.0" diff --git a/harness/src/provider-openai/main.ts b/harness/src/provider-openai/main.ts deleted file mode 100644 index e74efd934..000000000 --- a/harness/src/provider-openai/main.ts +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env node -import { bootstrapWorker } from '../runtime/worker.js'; -import { register } from './register.js'; - -await bootstrapWorker({ - name: 'provider-openai', - description: - 'OpenAI Chat Completions streaming provider on the iii bus (provider::openai::stream + ::complete).', - register: (iii, ctx) => register(iii, ctx), -}); diff --git a/harness/src/provider-openai/reasoning.ts b/harness/src/provider-openai/reasoning.ts deleted file mode 100644 index 8157ccd51..000000000 --- a/harness/src/provider-openai/reasoning.ts +++ /dev/null @@ -1,69 +0,0 @@ -/** - * OpenAI reasoning-effort selection. Detects reasoning models (catalog - * `supports_thinking` flag first, id pattern fallback) and maps the - * harness `thinking_level` onto a `reasoning_effort` valid for the model - * family, defaulting to `medium`. - */ - -const EFFORT_ORDER = ['none', 'minimal', 'low', 'medium', 'high', 'xhigh']; - -export function isReasoningModel(model: string, catalogSupportsThinking?: boolean): boolean { - if (typeof catalogSupportsThinking === 'boolean') return catalogSupportsThinking; - return /^(gpt-5|o1|o3|o4)/i.test(model); -} - -/** Efforts the model family accepts; empty = don't send the param. */ -function supportedEfforts(model: string): string[] { - const id = model.toLowerCase(); - if (!/^(gpt-5|o1|o3|o4)/.test(id)) return []; - // The o1 family (o1, o1-mini, o1-preview, o1-pro) rejects - // reasoning_effort on Chat Completions with a 400 — even though - // catalogs flag it as a reasoning model. Omit the param entirely. - if (id.startsWith('o1')) return []; - // Chat-tuned variants only support the fixed default; omit the param. - if (id.includes('chat')) return []; - // gpt-5-pro / gpt-5.x-pro: high only. - if (id.includes('pro')) return ['high']; - const minorMatch = id.match(/^gpt-5\.(\d+)/); - if (minorMatch) { - const minor = Number(minorMatch[1]); - // gpt-5.1: none/low/medium/high; gpt-5.2+ adds xhigh. - return minor >= 2 - ? ['none', 'low', 'medium', 'high', 'xhigh'] - : ['none', 'low', 'medium', 'high']; - } - // gpt-5 base family (mini/nano/codex). - if (id.startsWith('gpt-5')) return ['minimal', 'low', 'medium', 'high']; - // o-series. - return ['low', 'medium', 'high']; -} - -function normalizeLevel(level: string | undefined): string { - if (level === undefined) return 'medium'; - if (level === 'off') return 'none'; - if (level === 'max') return 'xhigh'; - return level; -} - -/** - * Effort for a reasoning model: the requested level when the family - * supports it, else the nearest supported effort below (then above). - * Returns undefined when the family takes no effort param. - */ -export function reasoningEffortFor(level: string | undefined, model: string): string | undefined { - const ladder = supportedEfforts(model); - if (ladder.length === 0) return undefined; - const want = normalizeLevel(level); - if (ladder.includes(want)) return want; - const wantIdx = EFFORT_ORDER.indexOf(want); - if (wantIdx < 0) return ladder.includes('medium') ? 'medium' : ladder[ladder.length - 1]; - for (let i = wantIdx - 1; i >= 0; i--) { - const candidate = EFFORT_ORDER[i]; - if (candidate !== undefined && ladder.includes(candidate)) return candidate; - } - for (let i = wantIdx + 1; i < EFFORT_ORDER.length; i++) { - const candidate = EFFORT_ORDER[i]; - if (candidate !== undefined && ladder.includes(candidate)) return candidate; - } - return undefined; -} diff --git a/harness/src/provider-openai/refresh-fn.ts b/harness/src/provider-openai/refresh-fn.ts deleted file mode 100644 index f1adf3cb3..000000000 --- a/harness/src/provider-openai/refresh-fn.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * `provider::openai::refresh_models` — re-pull the upstream model list and - * register the chat-capable subset into the iii models catalog. Returns - * `{ registered }`. Never throws across the bus boundary. - */ - -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import type { WorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; - -export const FUNCTION_ID = 'provider::openai::refresh_models'; - -export type RefreshResult = { registered: string[] }; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (): Promise => { - try { - return { registered: await discoverAndRegister(iii, worker) }; - } catch (err) { - logger.warn('provider::openai::refresh_models failed', { err: String(err) }); - return { registered: [] }; - } - }, - { - description: - 'Re-pull the OpenAI model list (GET /v1/models) and register the chat-capable subset into the iii models catalog. Idempotent.', - }, - ); -} diff --git a/harness/src/provider-openai/register.ts b/harness/src/provider-openai/register.ts deleted file mode 100644 index bee763aba..000000000 --- a/harness/src/provider-openai/register.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { loadConfig } from '../runtime/config.js'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import { declareProvider } from '../runtime/provider-resolve.js'; -import { PROVIDER_ID } from './auth.js'; -import { register as registerComplete } from './complete.js'; -import { loadWorkerConfig } from './config.js'; -import { discoverAndRegister } from './discover.js'; -import { register as registerRefresh } from './refresh-fn.js'; -import { register as registerStream } from './stream-fn.js'; - -export async function register(iii: ISdk, ctx: { configPath: string }): Promise { - const cfg = await loadConfig(ctx.configPath); - const worker = loadWorkerConfig(cfg); - registerComplete(iii, worker); - registerStream(iii, worker); - registerRefresh(iii, worker); - - void declareProvider(iii, { - id: PROVIDER_ID, - display_name: 'openai', - credential_env_var: 'OPENAI_API_KEY', - defaults: { - api_url: worker.default_api_url, - max_tokens: worker.default_max_tokens, - }, - supports_model_listing: true, - }); - - setImmediate(() => { - discoverAndRegister(iii, worker).catch((err) => { - logger.warn('openai startup discovery threw', { err: String(err) }); - }); - }); -} diff --git a/harness/src/provider-openai/sse.ts b/harness/src/provider-openai/sse.ts deleted file mode 100644 index c76982c1d..000000000 --- a/harness/src/provider-openai/sse.ts +++ /dev/null @@ -1,175 +0,0 @@ -/** - * OpenAI Chat Completions SSE state machine. Mirrors - * `provider-openai/crates/provider-base/src/openai_compat.rs::handle_chunk`. - */ - -import type { AssistantMessage } from '../types/agent-message.js'; -import type { ContentBlock } from '../types/content.js'; -import type { AssistantMessageEvent, ErrorKind, StopReason, Usage } from '../types/stream-event.js'; - -type PartialToolCall = { id: string; function_id: string; args_json: string }; - -export type PartialState = { - text: string; - tool_calls: PartialToolCall[]; - usage: Usage; - stop_reason: StopReason; -}; - -export function emptyPartial(): PartialState { - return { - text: '', - tool_calls: [], - usage: { input: 0, output: 0, cache_read: 0, cache_write: 0 }, - stop_reason: 'end', - }; -} - -function buildContent(state: PartialState): ContentBlock[] { - const out: ContentBlock[] = []; - if (state.text.length > 0) out.push({ type: 'text', text: state.text }); - for (const tc of state.tool_calls) { - if (tc.function_id.length === 0) continue; - let args: unknown = {}; - if (tc.args_json.length > 0) { - try { - args = JSON.parse(tc.args_json); - } catch { - args = null; - } - } - out.push({ type: 'function_call', id: tc.id, function_id: tc.function_id, arguments: args }); - } - return out; -} - -export function buildPartial( - state: PartialState, - model: string, - provider: string, -): AssistantMessage { - return { - role: 'assistant', - content: buildContent(state), - stop_reason: state.stop_reason, - error_message: null, - error_kind: null, - usage: state.usage, - model, - provider, - timestamp: Date.now(), - }; -} - -export function buildFinal(state: PartialState, model: string, provider: string): AssistantMessage { - return buildPartial(state, model, provider); -} - -export function mapFinishReason(s: string): StopReason { - if (s === 'stop') return 'end'; - if (s === 'length') return 'length'; - if (s === 'tool_calls' || s === 'function_call') return 'function_call'; - return 'end'; -} - -export function mergeUsage(usage: Record, into: Usage): void { - const num = (k: string) => (typeof usage[k] === 'number' ? (usage[k] as number) : 0); - into.input = (into.input ?? 0) + num('prompt_tokens') + num('input_tokens'); - into.output = (into.output ?? 0) + num('completion_tokens') + num('output_tokens'); - for (const parent of ['prompt_tokens_details', 'input_tokens_details']) { - const d = usage[parent] as Record | undefined; - if (d && typeof d.cached_tokens === 'number') { - into.cache_read = (into.cache_read ?? 0) + d.cached_tokens; - } - } -} - -export function classifyOpenaiError(message: string, status?: number): ErrorKind { - if (status === 401 || status === 403) return 'auth_expired'; - if (status === 429) return 'rate_limited'; - if (status && status >= 500) return 'transient'; - if (/context length|too many tokens/i.test(message)) return 'context_overflow'; - return 'permanent'; -} - -export function syntheticErrorEvent( - message: string, - model: string, - provider: string, - error_kind: ErrorKind = 'transient', -): AssistantMessageEvent { - const final: AssistantMessage = { - role: 'assistant', - content: [{ type: 'text', text: message }], - stop_reason: 'error', - error_message: message, - error_kind, - usage: null, - model, - provider, - timestamp: Date.now(), - }; - return { type: 'error', error: final }; -} - -/** - * Process one parsed Chat Completions chunk. Returns 0+ events. - * Returns `null` for the [DONE] sentinel — caller should break. - */ -export function handleChunk( - chunk: Record, - state: PartialState, - model: string, - provider: string, -): AssistantMessageEvent[] { - const events: AssistantMessageEvent[] = []; - const usage = chunk.usage as Record | undefined; - if (usage) mergeUsage(usage, state.usage); - const choices = chunk.choices; - if (!Array.isArray(choices) || choices.length === 0) return events; - const choice = choices[0] as Record; - const finish = typeof choice.finish_reason === 'string' ? choice.finish_reason : null; - if (finish) state.stop_reason = mapFinishReason(finish); - const delta = choice.delta as Record | undefined; - if (!delta) return events; - - if (typeof delta.content === 'string' && delta.content.length > 0) { - if (state.text.length === 0) { - events.push({ type: 'text_start', partial: buildPartial(state, model, provider) }); - } - state.text += delta.content; - events.push({ - type: 'text_delta', - partial: buildPartial(state, model, provider), - delta: delta.content, - }); - } - - const tool_calls = delta.tool_calls; - if (Array.isArray(tool_calls)) { - for (const tc of tool_calls) { - if (!tc || typeof tc !== 'object') continue; - const tcObj = tc as Record; - const index = typeof tcObj.index === 'number' ? tcObj.index : 0; - while (state.tool_calls.length <= index) { - state.tool_calls.push({ id: '', function_id: '', args_json: '' }); - } - const entry = state.tool_calls[index]; - if (!entry) continue; - if (typeof tcObj.id === 'string' && tcObj.id.length > 0) entry.id = tcObj.id; - const fn = tcObj.function as Record | undefined; - if (fn) { - if (typeof fn.name === 'string' && fn.name.length > 0) entry.function_id = fn.name; - if (typeof fn.arguments === 'string') { - entry.args_json += fn.arguments; - events.push({ - type: 'functioncall_delta', - partial: buildPartial(state, model, provider), - delta: fn.arguments, - }); - } - } - } - } - return events; -} diff --git a/harness/src/provider-openai/stream-fn.ts b/harness/src/provider-openai/stream-fn.ts deleted file mode 100644 index 102afb2f0..000000000 --- a/harness/src/provider-openai/stream-fn.ts +++ /dev/null @@ -1,58 +0,0 @@ -import type { ChannelWriter } from 'iii-sdk'; -import type { ISdk } from '../runtime/iii.js'; -import { logger } from '../runtime/otel.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import { - ProviderStreamInputJsonSchema, - ProviderStreamOutputJsonSchema, - ProviderStreamRuntimeInputSchema, -} from '../types/provider.js'; -import { isTerminal } from '../types/stream-event.js'; -import { buildConfig } from './auth.js'; -import type { WorkerConfig } from './config.js'; -import { streamOpenai } from './stream.js'; - -export const FUNCTION_ID = 'provider::openai::stream'; - -export function register(iii: ISdk, worker: WorkerConfig): void { - iii.registerFunction( - FUNCTION_ID, - async (raw: unknown) => { - const input = ProviderStreamRuntimeInputSchema.parse(raw); - // The iii-sdk auto-hydrates `writer_ref` (a StreamChannelRef on the - // wire) into a `ChannelWriter` instance before this handler runs — - // use it directly instead of re-instantiating from the wire shape - // (which no longer exists by the time we get here). - const writer = input.writer_ref as ChannelWriter; - const cfg = await buildConfig(iii, worker, input.model); - try { - const events = streamOpenai({ - cfg, - system_prompt: input.system_prompt ?? '', - messages: input.messages as AgentMessage[], - tools: input.tools as import('../types/function.js').AgentFunction[], - ...(input.thinking_level ? { thinking_level: input.thinking_level } : {}), - }); - for await (const ev of events) { - writer.sendMessage(JSON.stringify(ev)); - if (isTerminal(ev)) break; - } - } catch (err) { - logger.warn('provider::openai::stream failed mid-flight', { err: String(err) }); - } finally { - try { - writer.close(); - } catch (err) { - logger.debug('writer.close failed', { err: String(err) }); - } - } - return { ok: true }; - }, - { - description: - 'Stream a single assistant turn from OpenAI Chat Completions into the caller-supplied channel.', - request_format: ProviderStreamInputJsonSchema as Record, - response_format: ProviderStreamOutputJsonSchema as Record, - }, - ); -} diff --git a/harness/src/provider-openai/stream.ts b/harness/src/provider-openai/stream.ts deleted file mode 100644 index 70560ce8d..000000000 --- a/harness/src/provider-openai/stream.ts +++ /dev/null @@ -1,168 +0,0 @@ -/** - * OpenAI Chat Completions stream. Mirrors - * `provider-openai/crates/provider-base/src/openai_compat.rs::stream_chat_completions`. - */ - -import { logger } from '../runtime/otel.js'; -import type { AgentMessage, AssistantMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import type { AssistantMessageEvent } from '../types/stream-event.js'; -import { isReasoningModel, reasoningEffortFor } from './reasoning.js'; -import { - buildFinal, - classifyOpenaiError, - emptyPartial, - handleChunk, - syntheticErrorEvent, -} from './sse.js'; -import type { ChatCompletionsConfig } from './types.js'; -import { toOpenaiMessages } from './wire-messages.js'; -import { functionsToOpenai } from './wire-tools.js'; - -export type StreamArgs = { - cfg: ChatCompletionsConfig; - system_prompt: string; - messages: AgentMessage[]; - tools: AgentFunction[]; - /** Optional reasoning level; mapped onto `reasoning_effort` for reasoning models. */ - thinking_level?: string; -}; - -export async function* streamOpenai({ - cfg, - system_prompt, - messages, - tools, - thinking_level, -}: StreamArgs): AsyncGenerator { - // `max_completion_tokens` stays set for reasoning models too: reasoning - // tokens count toward it, but the clamped default leaves ample room. - const body: Record = { - model: cfg.model, - max_completion_tokens: cfg.max_tokens, - messages: toOpenaiMessages(messages, system_prompt), - stream: true, - stream_options: { include_usage: true }, - }; - if (isReasoningModel(cfg.model, cfg.catalog?.supports_thinking)) { - const effort = reasoningEffortFor(thinking_level, cfg.model); - if (effort) body.reasoning_effort = effort; - } - if (tools.length > 0) body.tools = functionsToOpenai(tools); - - const authName = cfg.auth_header_name ?? 'Authorization'; - const authPrefix = cfg.auth_value_prefix ?? 'Bearer '; - const headers: Record = { - 'content-type': 'application/json', - [authName]: `${authPrefix}${cfg.api_key}`, - }; - for (const [k, v] of cfg.extra_headers ?? []) headers[k] = v; - - let resp: Response; - try { - resp = await fetch(cfg.url, { - method: 'POST', - headers, - body: JSON.stringify(body), - }); - } catch (err) { - yield syntheticErrorEvent(`openai fetch failed: ${String(err)}`, cfg.model, cfg.provider_name); - return; - } - if (!resp.ok) { - const text = await resp.text().catch(() => ''); - yield syntheticErrorEvent( - text || `openai http ${resp.status}`, - cfg.model, - cfg.provider_name, - classifyOpenaiError(text, resp.status), - ); - return; - } - const partial: AssistantMessage = { - role: 'assistant', - content: [], - stop_reason: 'end', - error_message: null, - error_kind: null, - usage: null, - model: cfg.model, - provider: cfg.provider_name, - timestamp: Date.now(), - }; - yield { type: 'start', partial }; - - const state = emptyPartial(); - if (!resp.body) { - yield syntheticErrorEvent('openai response missing body', cfg.model, cfg.provider_name); - return; - } - const reader = resp.body.getReader(); - const decoder = new TextDecoder(); - let buf = ''; - try { - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - let idx = buf.indexOf('\n\n'); - while (idx >= 0) { - const block = buf.slice(0, idx); - buf = buf.slice(idx + 2); - const dataLine = parseDataLine(block); - idx = buf.indexOf('\n\n'); - if (dataLine === null) continue; - if (dataLine === '[DONE]') { - yield { type: 'done', message: buildFinal(state, cfg.model, cfg.provider_name) }; - return; - } - let parsed: Record | null = null; - try { - parsed = JSON.parse(dataLine) as Record; - } catch { - continue; - } - if (parsed) { - for (const e of handleChunk(parsed, state, cfg.model, cfg.provider_name)) yield e; - } - } - } - } catch (err) { - logger.warn('openai stream read failed', { err: String(err) }); - yield syntheticErrorEvent(`stream read failed: ${String(err)}`, cfg.model, cfg.provider_name); - return; - } - yield { type: 'done', message: buildFinal(state, cfg.model, cfg.provider_name) }; -} - -function parseDataLine(block: string): string | null { - let data: string | null = null; - for (const line of block.split('\n')) { - if (line.startsWith('data: ')) data = line.slice('data: '.length); - } - return data; -} - -export async function collect( - events: AsyncIterable, -): Promise { - let last: AssistantMessage | null = null; - for await (const ev of events) { - if (ev.type === 'done') return ev.message; - if (ev.type === 'error') return ev.error; - if ('partial' in ev) last = ev.partial; - } - return ( - last ?? { - role: 'assistant', - content: [], - stop_reason: 'error', - error_message: 'stream closed without final', - error_kind: 'transient', - usage: null, - model: 'openai', - provider: 'openai', - timestamp: Date.now(), - } - ); -} diff --git a/harness/src/provider-openai/types.ts b/harness/src/provider-openai/types.ts deleted file mode 100644 index 4591ef89d..000000000 --- a/harness/src/provider-openai/types.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { Model } from '../models-catalog/types.js'; -import type { Credential } from '../runtime/provider-resolve.js'; - -export type ChatCompletionsConfig = { - url: string; - provider_name: string; - model: string; - api_key: string; - /** Defaults to "Authorization". */ - auth_header_name?: string; - /** Defaults to "Bearer ". */ - auth_value_prefix?: string; - extra_headers?: Array; - max_tokens: number; - /** Catalog entry for `model` when known; used for reasoning detection. In-process only — never serialized, no Rust counterpart. */ - catalog?: Model; -}; - -export function configFromCredential( - url: string, - provider_name: string, - model: string, - cred: Credential, - max_tokens: number, -): ChatCompletionsConfig { - const api_key = cred.type === 'api_key' ? cred.key : cred.access_token; - return { url, provider_name, model, api_key, max_tokens }; -} diff --git a/harness/src/provider-openai/wire-messages.ts b/harness/src/provider-openai/wire-messages.ts deleted file mode 100644 index 2dcd51c11..000000000 --- a/harness/src/provider-openai/wire-messages.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** - * AgentMessage[] → OpenAI Chat Completions wire shape. Mirrors - * `provider-openai/crates/provider-base/src/openai_compat.rs::to_openai_messages`. - */ - -import type { AgentMessage } from '../types/agent-message.js'; -import { formatFunctionResultContent } from '../types/wire.js'; - -export function toOpenaiMessages(messages: AgentMessage[], system_prompt: string): unknown[] { - const out: unknown[] = []; - if (system_prompt.length > 0) { - out.push({ role: 'system', content: system_prompt }); - } - for (const m of messages) { - if (m.role === 'user') { - const text = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'text' }> => c.type === 'text', - ) - .map((c) => c.text) - .join('\n'); - out.push({ role: 'user', content: text }); - } else if (m.role === 'assistant') { - const text = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'text' }> => c.type === 'text', - ) - .map((c) => c.text) - .join('\n'); - const tool_calls = m.content - .filter( - (c): c is Extract<(typeof m.content)[number], { type: 'function_call' }> => - c.type === 'function_call', - ) - .map((c) => ({ - id: c.id, - type: 'function', - function: { name: c.function_id, arguments: JSON.stringify(c.arguments) }, - })); - const entry: Record = { role: 'assistant' }; - if (text.length > 0) entry.content = text; - if (tool_calls.length > 0) entry.tool_calls = tool_calls; - out.push(entry); - } else if (m.role === 'function_result') { - const text = formatFunctionResultContent(m); - const row: Record = { - role: 'tool', - tool_call_id: m.function_call_id, - content: text, - }; - if (m.is_error) row.is_error = true; - // Boundary dedup: never ship two tool messages with the same - // tool_call_id. Some OpenAI-compatible servers accept duplicates - // and silently overwrite; others (Anthropic's compat shim, - // strict gateways) reject. Latest-wins: replace any prior tool - // message with the same id rather than appending. - const existingIdx = out.findIndex( - (e) => - (e as { role?: string }).role === 'tool' && - (e as { tool_call_id?: string }).tool_call_id === m.function_call_id, - ); - if (existingIdx >= 0) { - out[existingIdx] = row; - } else { - out.push(row); - } - } - // custom messages are skipped - } - return out; -} diff --git a/harness/src/provider-openai/wire-tools.ts b/harness/src/provider-openai/wire-tools.ts deleted file mode 100644 index c133e4f33..000000000 --- a/harness/src/provider-openai/wire-tools.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { AgentFunction } from '../types/function.js'; - -export function functionsToOpenai(functions: AgentFunction[]): unknown[] { - return functions.map((t) => ({ - type: 'function', - function: { - name: t.name, - description: t.description, - parameters: t.parameters, - }, - })); -} diff --git a/harness/src/runtime/catalog.ts b/harness/src/runtime/catalog.ts new file mode 100644 index 000000000..d9360e24a --- /dev/null +++ b/harness/src/runtime/catalog.ts @@ -0,0 +1,39 @@ +/** + * Read helper for the llm-router model catalog. Output-token budgeting moved + * into the router itself (override > model ceiling > soft cap); the harness + * only reads catalog entries for preflight sizing and provisioning metadata. + */ + +import type { Model } from '../types/model.js'; +import type { ISdk } from './iii.js'; +import { logger } from './otel.js'; + +const MODELS_GET_TIMEOUT_MS = 5_000; + +/** + * Fetch the catalog entry for `(provider, model)` via `router::models::get` + * (payload key is `id`, the result is wrapped as `{ model }`, null on miss). + * Best-effort: returns null on miss, timeout, or any bus error — callers fall + * back to defaults so an empty catalog never breaks a request. + */ +export async function getCatalogModel( + iii: ISdk, + provider: string, + modelId: string, +): Promise { + try { + const entry = await iii.trigger({ + function_id: 'router::models::get', + payload: { provider, id: modelId }, + timeoutMs: MODELS_GET_TIMEOUT_MS, + }); + return entry?.model ?? null; + } catch (err) { + logger.debug('catalog: router::models::get failed', { + provider, + model: modelId, + err: String(err), + }); + return null; + } +} diff --git a/harness/src/runtime/harness-config.ts b/harness/src/runtime/harness-config.ts index 65c94b555..d6f3367af 100644 --- a/harness/src/runtime/harness-config.ts +++ b/harness/src/runtime/harness-config.ts @@ -1,22 +1,18 @@ /** * Shape of the single `harness` configuration entry that lives in the - * built-in `configuration` worker. It replaces the former `database`-backed - * `auth-credentials` (api keys) and `provider-config` (runtime overrides) - * tables, and adds a `permissions` block. + * built-in `configuration` worker — the agent `permissions` block only. + * Provider credentials/settings moved to the `llm-router` entry, whose + * schema the router composes from provider declarations + * (see `harness/migrate-llm-router-config.ts` for the one-time copy). * * Value shape: * * { - * "permissions": { "default_mode": "manual" | "auto" | "full" }, - * "providers": { - * "anthropic": { "api_key": "...", "api_url": "...", "max_tokens": 8192 }, - * ... - * } + * "permissions": { "default_mode": "manual" | "auto" | "full" } * } * - * The `providers` JSON Schema is composed dynamically from each provider's - * self-declared schema (see `harness/providers/registry.ts`), so adding a - * provider changes the editable shape automatically. + * A stale `providers` block from the pre-router layout may still be present + * in stored values; it is ignored here and tolerated by the entry schema. */ import { configurationGet, type JsonValue } from './configuration.js'; @@ -28,39 +24,25 @@ export const HARNESS_CONFIG_ID = 'harness'; export const PERMISSION_MODES = ['manual', 'auto', 'full'] as const; export type PermissionMode = (typeof PERMISSION_MODES)[number]; -/** One provider's stored config. Secret (`api_key`) + non-secret settings. */ -export type HarnessProviderConfig = { - api_key?: string; - api_url?: string; - max_tokens?: number; -} & Record; - export type HarnessPermissions = { default_mode: PermissionMode; }; export type HarnessConfigValue = { permissions: HarnessPermissions; - providers: Record; }; export const DEFAULT_PERMISSION_MODE: PermissionMode = 'manual'; -/** The value used to seed the entry the first time it is registered. */ -export function baseHarnessConfigValue(): HarnessConfigValue { - return { - permissions: { default_mode: DEFAULT_PERMISSION_MODE }, - providers: {}, - }; -} - function isPermissionMode(v: unknown): v is PermissionMode { return v === 'manual' || v === 'auto' || v === 'full'; } /** Coerce an arbitrary JSON value into a well-formed `HarnessConfigValue`. */ export function normalizeHarnessConfig(value: JsonValue | null): HarnessConfigValue { - const base = baseHarnessConfigValue(); + const base: HarnessConfigValue = { + permissions: { default_mode: DEFAULT_PERMISSION_MODE }, + }; if (!value || typeof value !== 'object' || Array.isArray(value)) return base; const obj = value as Record; @@ -69,15 +51,6 @@ export function normalizeHarnessConfig(value: JsonValue | null): HarnessConfigVa const mode = (permissions as Record).default_mode; if (isPermissionMode(mode)) base.permissions.default_mode = mode; } - - const providers = obj.providers; - if (providers && typeof providers === 'object' && !Array.isArray(providers)) { - for (const [id, cfg] of Object.entries(providers)) { - if (cfg && typeof cfg === 'object' && !Array.isArray(cfg)) { - base.providers[id] = cfg as HarnessProviderConfig; - } - } - } return base; } @@ -89,63 +62,3 @@ export async function readHarnessConfig(iii: ISdk): Promise const value = await configurationGet(iii, HARNESS_CONFIG_ID, { raw: false }); return normalizeHarnessConfig(value); } - -/** Fields that affect upstream model listing for a provider. */ -const DISCOVERY_FINGERPRINT_KEYS = ['api_key', 'api_url'] as const; - -function normalizeApiKey(v: unknown): string { - return typeof v === 'string' ? v : ''; -} - -/** - * Stable fingerprint of provider settings that should trigger model - * re-discovery when changed. - */ -export function providerDiscoveryFingerprint(cfg: HarnessProviderConfig): string { - const parts: string[] = []; - for (const k of DISCOVERY_FINGERPRINT_KEYS) { - parts.push(`${k}=${normalizeApiKey(cfg[k])}`); - } - return parts.join('\u0001'); -} - -/** - * Provider ids whose discovery-relevant config changed between two harness - * snapshots. Permissions-only edits return []. - */ -export function providersAffectedByConfigChange( - oldCfg: HarnessConfigValue, - newCfg: HarnessConfigValue, -): string[] { - const ids = new Set([...Object.keys(oldCfg.providers), ...Object.keys(newCfg.providers)]); - const affected: string[] = []; - for (const id of ids) { - const oldFp = providerDiscoveryFingerprint(oldCfg.providers[id] ?? {}); - const newFp = providerDiscoveryFingerprint(newCfg.providers[id] ?? {}); - if (oldFp !== newFp) affected.push(id); - } - return affected.sort(); -} - -export type ConfigurationChangeEvent = { - old_value: JsonValue | null; - new_value: JsonValue | null; -}; - -/** Parse a `configuration` trigger payload into old/new values when present. */ -export function parseConfigurationChangeEvent(payload: unknown): ConfigurationChangeEvent { - if (!payload || typeof payload !== 'object') { - return { old_value: null, new_value: null }; - } - const o = payload as Record; - const body = - o.new_value !== undefined || o.old_value !== undefined - ? o - : o.payload && typeof o.payload === 'object' - ? (o.payload as Record) - : o; - return { - old_value: (body.old_value ?? null) as JsonValue | null, - new_value: (body.new_value ?? null) as JsonValue | null, - }; -} diff --git a/harness/src/runtime/models-discovery.ts b/harness/src/runtime/models-discovery.ts deleted file mode 100644 index c66e2b6dd..000000000 --- a/harness/src/runtime/models-discovery.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Shared helpers for cloud-provider model discovery. Cloud providers - * (anthropic, openai, kimi) hit their upstream `/v1/models` endpoint, map - * the result onto the catalog `Model` shape with sane per-provider defaults, - * and register each into the iii models catalog via `models::reconcile`. - * - * This mirrors the local-provider discovery in - * `provider-lmstudio/discover.ts`, extended to remote APIs. The console - * model dropdown reads the cached result via `models::list`. - */ - -import type { Model } from '../models-catalog/types.js'; -import type { ISdk } from './iii.js'; -import type { ModelsDevModel } from './modelsdev.js'; -import { logger } from './otel.js'; - -const DISCOVERY_TIMEOUT_MS = 8_000; -const REGISTER_TIMEOUT_MS = 5_000; - -/** - * Derive a sibling endpoint URL from a chat/messages URL by swapping the - * trailing path segment, e.g. `.../v1/chat/completions` ->`.../v1/models` - * or `.../v1/messages` -> `.../v1/models`. Falls back to appending - * `/models` to the origin when the input doesn't match a known suffix. - */ -export function deriveModelsUrl(apiUrl: string): string { - const swapped = apiUrl.replace(/\/(chat\/completions|messages|completions)\/?$/, '/models'); - if (swapped !== apiUrl) return swapped; - try { - const u = new URL(apiUrl); - return `${u.protocol}//${u.host}/v1/models`; - } catch { - return `${apiUrl.replace(/\/+$/, '')}/models`; - } -} - -async function fetchWithTimeout( - url: string, - init: RequestInit, - timeoutMs: number, -): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), timeoutMs); - try { - return await fetch(url, { ...init, signal: controller.signal }); - } finally { - clearTimeout(timer); - } -} - -export type ModelsFetchResult = - | { kind: 'ok'; json: unknown } - | { kind: 'auth_error'; status: number } - | { kind: 'transient_error'; status?: number }; - -function isAuthStatus(status: number): boolean { - return status === 401 || status === 403; -} - -/** - * GET `url` for cloud provider model listing. Distinguishes invalid credentials - * (401/403) from transient failures so discovery can prune vs keep the catalog. - * Never throws across the bus boundary. - */ -export async function fetchModelsForDiscovery( - url: string, - headers: Record, -): Promise { - let resp: Response; - try { - resp = await fetchWithTimeout(url, { method: 'GET', headers }, DISCOVERY_TIMEOUT_MS); - } catch (err) { - logger.warn('model discovery: fetch failed', { url, err: String(err) }); - return { kind: 'transient_error' }; - } - if (!resp.ok) { - if (isAuthStatus(resp.status)) { - logger.info('model discovery: auth rejected', { url, status: resp.status }); - return { kind: 'auth_error', status: resp.status }; - } - logger.warn('model discovery: non-2xx response', { url, status: resp.status }); - return { kind: 'transient_error', status: resp.status }; - } - try { - return { kind: 'ok', json: await resp.json() }; - } catch (err) { - logger.warn('model discovery: invalid JSON', { url, err: String(err) }); - return { kind: 'transient_error' }; - } -} - -/** - * GET `url` and return the parsed JSON, or `null` on any error / non-2xx. - * Discovery is best-effort and must never throw across the bus boundary. - */ -export async function fetchModelsJson( - url: string, - headers: Record, -): Promise { - const result = await fetchModelsForDiscovery(url, headers); - return result.kind === 'ok' ? result.json : null; -} - -export type ModelStub = { - id: string; - display_name?: string; -}; - -/** - * Build a catalog `Model` for a discovered id. Upstream `/v1/models` - * endpoints expose little metadata, so per-model limits and capability - * flags come from models.dev when available (`modelsDev`), with a - * per-provider default context window and conservative flags as fallback. - */ -export function enrichModel(opts: { - provider: string; - api: string; - stub: ModelStub; - defaultContextWindow: number; - modelsDev?: ModelsDevModel; -}): Model { - const md = opts.modelsDev; - const model: Model = { - id: opts.stub.id, - provider: opts.provider, - api: opts.api, - display_name: opts.stub.display_name ?? opts.stub.id, - context_window: md?.limit?.context ?? opts.defaultContextWindow, - max_output_tokens: md?.limit?.output ?? 8_192, - supports_tools: md?.tool_call ?? true, - transports: ['sse'], - }; - if (md?.reasoning !== undefined) model.supports_thinking = md.reasoning; - return model; -} - -/** - * Replace the provider's catalog with `models` in one `models::reconcile` call - * (single state write). Best-effort: failures are logged and swallowed so - * discovery never throws across the bus boundary. - */ -export async function reconcileModels( - iii: ISdk, - provider: string, - models: readonly Model[], -): Promise { - try { - const res = await iii.trigger({ - function_id: 'models::reconcile', - payload: { provider, models: [...models] }, - timeoutMs: REGISTER_TIMEOUT_MS, - }); - if (Array.isArray(res?.ids) && res.ids.length > 0) return res.ids; - return models.map((m) => m.id); - } catch (err) { - logger.warn('model discovery: reconcile failed', { provider, err: String(err) }); - return []; - } -} diff --git a/harness/src/runtime/modelsdev.ts b/harness/src/runtime/modelsdev.ts deleted file mode 100644 index 9231f1516..000000000 --- a/harness/src/runtime/modelsdev.ts +++ /dev/null @@ -1,169 +0,0 @@ -/** - * models.dev catalog client. - * - * Fetches https://models.dev/api.json once per process (1h TTL, in-flight - * dedup) and exposes per-provider/model limits + capability flags used to - * enrich discovery. Best-effort by contract: any failure yields an empty - * index so discovery proceeds with its existing per-provider defaults. - */ - -import { logger } from './otel.js'; - -export type ModelsDevLimit = { - context?: number; - input?: number; - output?: number; -}; - -export type ModelsDevModel = { - id: string; - limit?: ModelsDevLimit; - reasoning?: boolean; - tool_call?: boolean; -}; - -/** providerKey -> (modelId -> ModelsDevModel) */ -export type ModelsDevIndex = Map>; - -export const MODELSDEV_URL = 'https://models.dev/api.json'; -const FETCH_TIMEOUT_MS = 8_000; -const CACHE_TTL_MS = 60 * 60 * 1000; -// Failed fetches cache an empty index, but only briefly — a transient blip -// at startup shouldn't strip catalog limits for a full hour. -const FAILURE_TTL_MS = 5 * 60 * 1000; -// Sanity bounds for externally-sourced token limits. models.dev is a -// third-party feed; an absurd value (e.g. output: 1) would silently break -// every request to that model, so out-of-range values are treated as absent. -const LIMIT_MIN = 256; -const LIMIT_MAX = 50_000_000; - -/** Harness provider id -> models.dev provider key. Local providers have no entry. */ -export const PROVIDER_KEY_MAP: Record = { - anthropic: 'anthropic', - openai: 'openai', - kimi: 'moonshotai', -}; - -type CacheEntry = { index: ModelsDevIndex; fetchedAt: number }; - -let cache: CacheEntry | null = null; -let inflight: Promise | null = null; - -/** Test hook: clear the module cache. */ -export function _resetModelsDevForTests(): void { - cache = null; - inflight = null; -} - -function emptyIndex(): ModelsDevIndex { - return new Map(); -} - -function num(v: unknown): number | undefined { - return typeof v === 'number' && Number.isFinite(v) && v >= LIMIT_MIN && v <= LIMIT_MAX - ? v - : undefined; -} - -function parseIndex(json: unknown): ModelsDevIndex { - const index = emptyIndex(); - if (typeof json !== 'object' || json === null) return index; - for (const [providerKey, provider] of Object.entries(json as Record)) { - if (typeof provider !== 'object' || provider === null) continue; - const models = (provider as Record).models; - if (typeof models !== 'object' || models === null) continue; - const byId = new Map(); - for (const [modelId, raw] of Object.entries(models as Record)) { - if (typeof raw !== 'object' || raw === null) continue; - const m = raw as Record; - const limit = - typeof m.limit === 'object' && m.limit !== null - ? (m.limit as Record) - : undefined; - byId.set(modelId, { - id: modelId, - limit: limit - ? { context: num(limit.context), input: num(limit.input), output: num(limit.output) } - : undefined, - reasoning: typeof m.reasoning === 'boolean' ? m.reasoning : undefined, - tool_call: typeof m.tool_call === 'boolean' ? m.tool_call : undefined, - }); - } - if (byId.size > 0) index.set(providerKey, byId); - } - return index; -} - -async function fetchIndex(): Promise { - const ac = new AbortController(); - const timer = setTimeout(() => ac.abort(), FETCH_TIMEOUT_MS); - try { - const resp = await fetch(MODELSDEV_URL, { signal: ac.signal }); - if (!resp.ok) { - logger.warn('modelsdev: non-200 response', { status: resp.status }); - return emptyIndex(); - } - const json: unknown = await resp.json(); - const index = parseIndex(json); - logger.info('modelsdev: catalog fetched', { providers: index.size }); - return index; - } catch (err) { - logger.warn('modelsdev: fetch failed', { err: String(err) }); - return emptyIndex(); - } finally { - clearTimeout(timer); - } -} - -/** - * Cached models.dev index. Never throws — failures cache an empty index - * (with a shorter TTL) so a flaky network can't hammer models.dev or block - * discovery, while a transient blip recovers within minutes. - */ -export async function getModelsDevIndex(): Promise { - if (cache) { - const ttl = cache.index.size > 0 ? CACHE_TTL_MS : FAILURE_TTL_MS; - if (Date.now() - cache.fetchedAt < ttl) return cache.index; - } - if (inflight) return inflight; - inflight = fetchIndex().then((index) => { - cache = { index, fetchedAt: Date.now() }; - inflight = null; - return index; - }); - return inflight; -} - -/** Strip a trailing `-YYYYMMDD` date suffix (`claude-sonnet-4-20250514` -> `claude-sonnet-4`). */ -function stripDateSuffix(id: string): string { - return id.replace(/-\d{8}$/, ''); -} - -/** - * Look up a model's models.dev metadata for a harness provider id. - * Conservative matching: exact id, then date-suffix-normalized on both sides. - * No match -> undefined (caller keeps its defaults). - */ -export function lookupModelsDev( - index: ModelsDevIndex, - providerId: string, - modelId: string, -): ModelsDevModel | undefined { - const key = PROVIDER_KEY_MAP[providerId]; - if (!key) return undefined; - const models = index.get(key); - if (!models) return undefined; - - const exact = models.get(modelId); - if (exact) return exact; - - const normalized = stripDateSuffix(modelId); - if (normalized !== modelId) { - const byNormalized = models.get(normalized); - if (byNormalized) return byNormalized; - } - for (const [id, model] of models) { - if (stripDateSuffix(id) === normalized) return model; - } - return undefined; -} diff --git a/harness/src/runtime/openai-compat-url.ts b/harness/src/runtime/openai-compat-url.ts deleted file mode 100644 index 130349545..000000000 --- a/harness/src/runtime/openai-compat-url.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Shared URL normalisation for OpenAI-compatible local-server providers - * (lmstudio, llamacpp). Both providers accept a base URL (`host:port`) - * or a full chat-completions URL; we always POST to - * `/v1/chat/completions`, so a missing path here means the request - * lands on the server root and llama-server / LM Studio return 404. - * - * `resolveApiUrl` in each provider's `config.ts` historically applied - * this auto-append only to the `*_BASE_URL` env var path -- the - * config.yaml `default_api_url` was assumed to already be fully - * qualified, and the runtime override stored under - * `provider_config::set` was used verbatim. That left a sharp edge: - * users who saved a base URL (e.g. `http://192.168.1.206:8080`) from - * the Providers UI got 404s on every request. This helper closes the - * gap so the override path normalises the same way as the env-var - * path. - */ - -/** Return `raw` with `/v1/chat/completions` appended if missing, or `null` if `raw` is not a usable http(s) URL. */ -export function normalizeChatCompletionsUrl(raw: string): string | null { - const trimmed = raw.trim(); - if (trimmed.length === 0) return null; - // Reject inputs that don't have a usable origin before we go appending - // a path -- `new URL("http://")` parses but with hostname "", which - // would then concat into the absurd `http://v1/chat/completions`. - let base: URL; - try { - base = new URL(trimmed); - } catch { - return null; - } - if (base.protocol !== 'http:' && base.protocol !== 'https:') return null; - if (base.hostname.length === 0) return null; - - // Work in URL space — string concatenation on the raw input mishandles - // query-string-only inputs. `http://host?foo=bar` would otherwise become - // `http://host?foo=bar/v1/chat/completions`, with the path segment buried - // inside the query string and every request hitting the server root. - const pathNoTrailing = base.pathname.replace(/\/+$/, ''); - if (pathNoTrailing.endsWith('/chat/completions')) { - base.pathname = pathNoTrailing; - } else { - base.pathname = `${pathNoTrailing}/v1/chat/completions`; - } - return base.toString(); -} diff --git a/harness/src/runtime/output-tokens.ts b/harness/src/runtime/output-tokens.ts deleted file mode 100644 index efc1c0fbb..000000000 --- a/harness/src/runtime/output-tokens.ts +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Per-request output-token resolution. - * - * The default request budget is `min(model.max_output_tokens, OUTPUT_TOKEN_MAX)` - * so high-output models (Claude 64k/128k, GPT-5 272k) don't burn latency/cost - * on every turn. An explicit registry override always wins, clamped only to the - * model's hard ceiling (never raised) to avoid upstream 400s. - */ - -import type { Model } from '../models-catalog/types.js'; -import type { ISdk } from './iii.js'; -import { logger } from './otel.js'; - -export const OUTPUT_TOKEN_MAX = 32_000; -export const OUTPUT_TOKEN_MAX_ENV = 'HARNESS_OUTPUT_TOKEN_MAX'; - -const MODELS_GET_TIMEOUT_MS = 5_000; - -let cachedCap: number | null = null; - -/** Cap from `HARNESS_OUTPUT_TOKEN_MAX` (strict positive integer) or 32_000. Cached per process. */ -export function outputTokenCap(): number { - if (cachedCap !== null) return cachedCap; - const v = process.env[OUTPUT_TOKEN_MAX_ENV]; - if (!v) { - cachedCap = OUTPUT_TOKEN_MAX; - return cachedCap; - } - // Strict digits-only parse: parseInt would silently truncate values like - // "1e9" to 1, turning a fat-fingered env var into a 1-token output cap. - const n = /^\d+$/.test(v.trim()) ? Number.parseInt(v.trim(), 10) : Number.NaN; - cachedCap = Number.isFinite(n) && n > 0 ? n : OUTPUT_TOKEN_MAX; - return cachedCap; -} - -/** Test hook: clear the cached env cap. */ -export function _resetOutputTokenCapForTests(): void { - cachedCap = null; -} - -export type ClampArgs = { - /** Catalog `max_output_tokens` for the model (undefined/0 = unknown). */ - modelMaxOutput: number | null | undefined; - /** Explicit registry override (`harness::provider::resolve` max_tokens). */ - userOverride: number | null; - /** Provider worker default (`default_max_tokens`, 8192). */ - workerDefault: number; - /** Defaults to `outputTokenCap()`. */ - cap?: number; -}; - -/** - * Resolve the per-request max output tokens. - * - * Precedence: - * 1. `userOverride > 0` — wins; clamped down to the model ceiling when known - * (deliberate choice is honored, so the 32k cap does NOT apply here). - * 2. `min(modelMaxOutput, cap)` — the per-model default. - * 3. `workerDefault` — unknown model, preserves pre-clamp behavior. - */ -export function clampOutputTokens(args: ClampArgs): number { - const cap = args.cap ?? outputTokenCap(); - const modelMax = - typeof args.modelMaxOutput === 'number' && args.modelMaxOutput > 0 - ? args.modelMaxOutput - : undefined; - - // Floored: token counts must be integers upstream. - if (typeof args.userOverride === 'number' && args.userOverride > 0) { - return Math.floor( - modelMax !== undefined ? Math.min(args.userOverride, modelMax) : args.userOverride, - ); - } - if (modelMax !== undefined) return Math.floor(Math.min(modelMax, cap)); - return Math.floor(args.workerDefault); -} - -/** - * Fetch the catalog entry for `(provider, model)` via `models::get`. - * Best-effort: returns null on miss, timeout, or any bus error — callers fall - * back to worker defaults so an empty catalog never breaks a request. - */ -export async function getCatalogModel( - iii: ISdk, - provider: string, - modelId: string, -): Promise { - try { - const entry = await iii.trigger({ - function_id: 'models::get', - payload: { provider, model_id: modelId }, - timeoutMs: MODELS_GET_TIMEOUT_MS, - }); - return entry ?? null; - } catch (err) { - logger.debug('output-tokens: models::get failed', { - provider, - model: modelId, - err: String(err), - }); - return null; - } -} diff --git a/harness/src/runtime/provider-resolve.ts b/harness/src/runtime/provider-resolve.ts deleted file mode 100644 index 4236e5a51..000000000 --- a/harness/src/runtime/provider-resolve.ts +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Provider-side helpers for the harness provider registry: - * - * - `declareProvider` announces a provider (id, defaults, env var) at - * startup so the harness composes it into the dynamic `harness` - * configuration schema. - * - `resolveProvider` fetches the credential + settings (api_url, - * max_tokens) at request time, replacing the old `auth::get_token` + - * `provider_config::get` pair with one call. - * - * Declaration is best-effort (logged on failure — the provider still serves - * with its config.yaml defaults and the env var declared via - * `credential_env_var`). - * - * This module also owns the `Credential` shape returned by - * `harness::provider::resolve` (the resolved api key / oauth token). - */ - -import type { ISdk } from './iii.js'; -import { logger } from './otel.js'; - -export type ApiKeyCredential = { - type: 'api_key'; - key: string; -}; - -export type OAuthCredential = { - type: 'oauth'; - access_token: string; - refresh_token?: string; - expires_at?: number; - scopes?: string[]; - provider_extra?: unknown; -}; - -export type Credential = ApiKeyCredential | OAuthCredential; - -export type ProviderDeclaration = { - id: string; - display_name?: string; - /** Env var consulted as a credential fallback when no `api_key` is configured. */ - credential_env_var?: string; - /** Optional explicit JSON Schema; when omitted the harness derives one from `defaults`. */ - config_schema?: Record; - defaults?: { api_url?: string; max_tokens?: number } & Record; - supports_model_listing?: boolean; -}; - -export type ProviderResolveResult = { - configured: boolean; - source: 'stored' | 'environment' | 'runtime' | 'fallback' | null; - credential: Credential | null; - api_url: string | null; - max_tokens: number | null; -}; - -const DECLARE_TIMEOUT_MS = 5_000; -const RESOLVE_TIMEOUT_MS = 5_000; - -export async function declareProvider(iii: ISdk, decl: ProviderDeclaration): Promise { - try { - await iii.trigger({ - function_id: 'harness::provider::register', - payload: decl, - timeoutMs: DECLARE_TIMEOUT_MS, - }); - } catch (err) { - logger.warn('provider declare failed', { provider: decl.id, err: String(err) }); - } -} - -export async function resolveProvider(iii: ISdk, provider: string): Promise { - const res = await iii.trigger({ - function_id: 'harness::provider::resolve', - payload: { provider }, - timeoutMs: RESOLVE_TIMEOUT_MS, - }); - if (!res || typeof res !== 'object') { - return { configured: false, source: null, credential: null, api_url: null, max_tokens: null }; - } - return res; -} diff --git a/harness/src/runtime/worker.ts b/harness/src/runtime/worker.ts index 1c7b0a8cd..f0956a3ce 100644 --- a/harness/src/runtime/worker.ts +++ b/harness/src/runtime/worker.ts @@ -1,6 +1,6 @@ /** * Common worker bootstrap. Ports the shape used by every Rust worker's - * `main.rs` (`provider-anthropic/src/main.rs`, etc.): + * `main.rs` (`llm-router/src/main.rs`, etc.): * * - parse CLI flags (`--config`, `--url`, `--manifest`) * - call `registerWorker(url, { workerName, ... })` diff --git a/harness/src/turn-orchestrator/assistant-streaming/ports.ts b/harness/src/turn-orchestrator/assistant-streaming/ports.ts index 9b8c358e1..7884c34c1 100644 --- a/harness/src/turn-orchestrator/assistant-streaming/ports.ts +++ b/harness/src/turn-orchestrator/assistant-streaming/ports.ts @@ -3,7 +3,6 @@ */ import { z } from 'zod'; -import type { Model } from '../../models-catalog/types.js'; import { logger } from '../../runtime/otel.js'; import type { ISdk } from '../../runtime/iii.js'; import { sessionAppendMessage, sessionUpdateMessage } from '../../runtime/session.js'; @@ -15,26 +14,31 @@ import { import type { ContentBlock } from '../../types/content.js'; import type { AgentFunction } from '../../types/function.js'; import type { AssistantMessageEvent } from '../../types/stream-event.js'; -import { AgentFunctionSchema } from '../../types/provider.js'; +import { AgentFunctionSchema } from '../../types/function.js'; import { emit } from '../events.js'; import { runPreflight } from '../preflight.js'; -import { buildInput, targetFunctionId, type RouteDecision } from '../provider-router.js'; import { streamProviderTurn } from '../provider-stream.js'; import type { RunRequest } from '../run-request.js'; +import type { Model } from '../../types/model.js'; import { createTurnStatePorts, type TurnStatePorts } from '../state-runtime/ports.js'; export type StreamContext = { session_id: string; - decision: RouteDecision; + /** Routed provider from provisioning; '' lets the router decide server-side. */ + provider: string; + model: string; system_prompt: string; tools: AgentFunction[]; messages: AgentMessage[]; - /** Optional reasoning/thinking level from the run request. Absent = off. */ + /** Optional reasoning/thinking level from the run request. Absent or 'off' = off. */ thinking_level?: string; - /** Turn's pre-resolved catalog entry, threaded to the provider. Absent = off. */ - model_meta?: Model; - /** Turn's stable id (run start time), so the provider dedupes credential resolution. */ - resolution_key?: number; + /** + * Deterministic id for this turn's router request + * (`${session_id}:${started_at_ms}`): `run::abort` recomputes it for + * `router::abort`, and the router threads it to providers as the + * credential-resolve dedup key. + */ + request_id: string; }; export type StreamTurnOutcome = { @@ -98,7 +102,7 @@ export type AssistantStreamingPorts = TurnStatePorts & { appendAssistantPlaceholder( session_id: string, entry_id: string, - decision: RouteDecision, + route: Pick, origin: Record, ): Promise; /** @@ -125,20 +129,23 @@ export function createStreamingPorts(iii: ISdk): AssistantStreamingPorts { }, async streamTurn(ctx, onDelta) { + // 'off' is a run-request convention, not a router thinking level — + // the documented contract is omission. + const thinking = ctx.thinking_level && ctx.thinking_level !== 'off'; const { final, error } = await streamProviderTurn(iii, { session_id: ctx.session_id, - targetFn: targetFunctionId(ctx.decision), - buildInput: (writerRef) => - buildInput( - ctx.decision, - writerRef, - ctx.system_prompt, - ctx.messages, - ctx.tools, - ctx.thinking_level, - ctx.model_meta, - ctx.resolution_key, - ), + targetFn: 'router::chat', + buildInput: (writerRef) => ({ + writer_ref: writerRef, + request_id: ctx.request_id, + model: ctx.model, + ...(ctx.provider ? { provider: ctx.provider } : {}), + system_prompt: ctx.system_prompt, + messages: ctx.messages, + tools: ctx.tools, + ...(thinking ? { thinking_level: ctx.thinking_level } : {}), + metadata: { session_id: ctx.session_id }, + }), onDelta, }); return { final, error }; @@ -152,10 +159,10 @@ export function createStreamingPorts(iii: ISdk): AssistantStreamingPorts { }); }, - async appendAssistantPlaceholder(session_id, entry_id, decision, origin) { + async appendAssistantPlaceholder(session_id, entry_id, route, origin) { await sessionAppendMessage(iii, { session_id, - message: emptyAssistant(decision.provider, decision.model), + message: emptyAssistant(route.provider, route.model), entry_id, origin, }); diff --git a/harness/src/turn-orchestrator/assistant-streaming/run.ts b/harness/src/turn-orchestrator/assistant-streaming/run.ts index 85fabd579..d73a0c72b 100644 --- a/harness/src/turn-orchestrator/assistant-streaming/run.ts +++ b/harness/src/turn-orchestrator/assistant-streaming/run.ts @@ -10,7 +10,6 @@ import { assistantEntryId, runKey } from '../../runtime/session.js'; import type { AssistantMessage } from '../../types/agent-message.js'; -import { decide } from '../provider-router.js'; import { syntheticAssistant } from '../synthetic-assistant.js'; import { emitTurnEndOnce, transitionToFinishing } from '../state-runtime/turn-end.js'; import { enterFunctionExecute } from '../function-execute/run.js'; @@ -49,26 +48,33 @@ export async function prepareStreamContext( const loadOpts = { excludeEntryIds: [assistantEntry] }; let messages = await ports.loadMessages(rec.session_id, loadOpts); const { provider, model, system_prompt, function_schemas, thinking_level } = request; - const decision = decide({ provider, model }); + // Provisioning pinned the routed provider; fall back to the raw request + // provider for records provisioned before the router cutover. + const routedProvider = request.routed_provider ?? provider; const tools = parseFunctionSchemas(function_schemas); const model_meta = rec.model_meta; if ( - (await ports.runPreflight(rec.session_id, messages, decision.provider, model, model_meta)) === - 'compacted' + (await ports.runPreflight( + rec.session_id, + messages, + routedProvider || provider, + model, + model_meta, + )) === 'compacted' ) { messages = await ports.loadMessages(rec.session_id, loadOpts); } return { session_id: rec.session_id, - decision, + provider: routedProvider, + model, system_prompt, tools, messages, ...(thinking_level ? { thinking_level } : {}), - ...(model_meta ? { model_meta } : {}), - resolution_key: rec.started_at_ms, + request_id: `${rec.session_id}:${rec.started_at_ms}`, }; } @@ -104,16 +110,19 @@ export async function runStreamTurn( export function resolveAssistantMessage( outcome: StreamTurnOutcome, - decision: StreamContext['decision'], + ctx: Pick, ): AssistantMessage { if (outcome.final) return outcome.final; + // Defense-in-depth behind the router's terminal-frame guarantee: the + // router itself can die mid-relay, so a close-without-terminal still + // synthesizes a visible error. const reason = outcome.error ?? 'provider channel closed without final'; return syntheticAssistant({ stop_reason: 'error', text: reason, - provider: decision.provider, - model: decision.model, + provider: ctx.provider, + model: ctx.model, }); } @@ -169,14 +178,14 @@ export async function runAssistantStreaming( beginTurn(rec); const assistantEntry = turnAssistantEntryId(rec); const ctx = await prepareStreamContext(ports, rec, assistantEntry); - await ports.appendAssistantPlaceholder(rec.session_id, assistantEntry, ctx.decision, { + await ports.appendAssistantPlaceholder(rec.session_id, assistantEntry, ctx, { turn: rec.turn_count, }); const outcome = await runStreamTurn(ports, rec.session_id, assistantEntry, ctx); // When the provider died without a final, resolveAssistantMessage builds a // synthetic error assistant; its text lands on the entry via the strict // final update in finalizeAssistantTurn (no separate delta emission). - const asst = resolveAssistantMessage(outcome, ctx.decision); + const asst = resolveAssistantMessage(outcome, ctx); rec.last_assistant = asst; rec.assistant_body_streamed = outcome.body_streamed; diff --git a/harness/src/turn-orchestrator/preflight.ts b/harness/src/turn-orchestrator/preflight.ts index fd8236016..6065d4268 100644 --- a/harness/src/turn-orchestrator/preflight.ts +++ b/harness/src/turn-orchestrator/preflight.ts @@ -10,7 +10,7 @@ import { fetchModelLimit, limitFromModel } from '../context-compaction/model-resolver.js'; import { usable as computeUsable } from '../context-compaction/overflow.js'; -import type { Model } from '../models-catalog/types.js'; +import type { Model } from '../types/model.js'; import type { ISdk } from '../runtime/iii.js'; import { logger } from '../runtime/otel.js'; import { readActivePath } from '../runtime/session.js'; diff --git a/harness/src/turn-orchestrator/prompt/index.ts b/harness/src/turn-orchestrator/prompt/index.ts index 88f94a78b..bf70eb8d6 100644 --- a/harness/src/turn-orchestrator/prompt/index.ts +++ b/harness/src/turn-orchestrator/prompt/index.ts @@ -1,9 +1,10 @@ /** - * Per-model identity prompts, selected by the run's provider/model. - * Routing reuses the provider-router's family heuristics. + * Per-model identity prompts, selected by the run's ROUTED provider. + * Routing authority lives in the llm-router worker: provisioning resolves the + * provider once via `router::route` and persists it on the run request; this + * module is a pure provider → family lookup with no routing logic of its own. */ -import { decide } from '../provider-router.js'; import { PROMPT_ANTHROPIC } from './anthropic.js'; import { PROMPT_DEFAULT } from './default.js'; import { PROMPT_GPT } from './gpt.js'; @@ -18,20 +19,24 @@ const FAMILY_PROMPTS: Record = { default: PROMPT_DEFAULT, }; -export function promptFamily(provider: string, model: string): PromptFamily { - const route = decide({ provider, model }); - switch (route.provider) { +export function promptFamily(provider: string): PromptFamily { + switch (provider) { case 'anthropic': return 'anthropic'; case 'openai': return 'gpt'; case 'kimi': return 'kimi'; + // No routed provider (router unreachable during provisioning): mirror the + // router's seeded default_provider so the un-routed prompt matches what + // the routed turn would have served. + case '': + return 'anthropic'; default: return 'default'; } } -export function selectIdentityPrompt(provider: string, model: string): string { - return FAMILY_PROMPTS[promptFamily(provider, model)]; +export function selectIdentityPrompt(provider: string): string { + return FAMILY_PROMPTS[promptFamily(provider)]; } diff --git a/harness/src/turn-orchestrator/provider-router.ts b/harness/src/turn-orchestrator/provider-router.ts deleted file mode 100644 index efa715163..000000000 --- a/harness/src/turn-orchestrator/provider-router.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Provider-router as a pure library (per PHASE-2-PLAN.md §4). The - * orchestrator imports it directly; there is no `router::*` bus surface. - */ - -import type { Model } from '../models-catalog/types.js'; -import type { AgentMessage } from '../types/agent-message.js'; -import type { AgentFunction } from '../types/function.js'; -import type { ProviderStreamInput, StreamChannelRef } from '../types/provider.js'; - -export type RouteDecision = - | { provider: 'anthropic'; model: string } - | { provider: 'openai'; model: string } - | { provider: 'kimi'; model: string } - | { provider: 'lmstudio'; model: string } - | { provider: 'llamacpp'; model: string }; - -export type RouteRequest = { - provider?: string; - model: string; -}; - -/** Pick a provider for a request. Defaults to Anthropic when ambiguous. */ -export function decide(req: RouteRequest): RouteDecision { - const p = (req.provider ?? '').toLowerCase(); - if (p === 'openai') return { provider: 'openai', model: req.model }; - if (p === 'kimi') return { provider: 'kimi', model: req.model }; - if (p === 'lmstudio') return { provider: 'lmstudio', model: req.model }; - if (p === 'llamacpp') return { provider: 'llamacpp', model: req.model }; - if (!p && /^gpt-|^o\d-/i.test(req.model)) { - return { provider: 'openai', model: req.model }; - } - if (!p && /^kimi-|^moonshot-v1-/i.test(req.model)) { - return { provider: 'kimi', model: req.model }; - } - return { provider: 'anthropic', model: req.model }; -} - -export function targetFunctionId(d: RouteDecision): string { - switch (d.provider) { - case 'anthropic': - return 'provider::anthropic::stream'; - case 'openai': - return 'provider::openai::stream'; - case 'kimi': - return 'provider::kimi::stream'; - case 'lmstudio': - return 'provider::lmstudio::stream'; - case 'llamacpp': - return 'provider::llamacpp::stream'; - } -} - -export function buildInput( - d: RouteDecision, - writer_ref: StreamChannelRef, - system_prompt: string | null | undefined, - messages: AgentMessage[], - tools: AgentFunction[], - thinking_level?: string, - model_meta?: Model, - resolution_key?: number, -): ProviderStreamInput { - return { - writer_ref, - system_prompt: system_prompt ?? null, - model: d.model, - messages: messages as unknown[], - tools, - ...(thinking_level ? { thinking_level } : {}), - ...(model_meta ? { model_meta } : {}), - ...(resolution_key !== undefined ? { resolution_key } : {}), - }; -} diff --git a/harness/src/turn-orchestrator/provider-stream.ts b/harness/src/turn-orchestrator/provider-stream.ts index 95677c3a7..8b6983764 100644 --- a/harness/src/turn-orchestrator/provider-stream.ts +++ b/harness/src/turn-orchestrator/provider-stream.ts @@ -1,21 +1,26 @@ /** - * Provider streaming. Turns an iii stream channel plus the provider trigger into - * a single final `AssistantMessage`, hiding the pull-based message pump behind an - * async iterator. + * Streaming pump for `router::chat`. Turns an iii stream channel plus the + * router trigger into a single final `AssistantMessage`, hiding the + * pull-based message pump behind an async iterator. * - * `streamProviderTurn` owns channel creation, the concurrent provider trigger, - * and the read loop. The caller supplies how to build the provider input (it - * needs the channel's writer ref) and a per-delta callback used to emit UI + * `streamProviderTurn` owns channel creation, the concurrent trigger, and the + * read loop. The caller supplies how to build the request payload (it needs + * the channel's writer ref) and a per-delta callback used to emit UI * `message_update` events. */ import type { ISdk, StreamChannelRef } from '../runtime/iii.js'; import { logger } from '../runtime/otel.js'; import type { AssistantMessage } from '../types/agent-message.js'; -import type { ProviderStreamInput } from '../types/provider.js'; import type { AssistantMessageEvent } from '../types/stream-event.js'; -const PROVIDER_STREAM_TIMEOUT_MS = 300_000; +/** + * Outer trigger budget. Must exceed the router's own 300s stream budget so + * the router (which owns retries and terminal-frame synthesis) always + * finishes first — a shorter outer timeout would kill the bus call while the + * router still owns the stream. + */ +const PROVIDER_STREAM_TIMEOUT_MS = 320_000; type Channel = Awaited>; @@ -99,7 +104,7 @@ export async function streamProviderTurn( params: { session_id: string; targetFn: string; - buildInput: (writerRef: StreamChannelRef) => ProviderStreamInput; + buildInput: (writerRef: StreamChannelRef) => unknown; onDelta: (partial: AssistantMessage, event: AssistantMessageEvent) => Promise; }, ): Promise { diff --git a/harness/src/turn-orchestrator/provisioning/ports.ts b/harness/src/turn-orchestrator/provisioning/ports.ts index 7e406f2ec..694e0afa0 100644 --- a/harness/src/turn-orchestrator/provisioning/ports.ts +++ b/harness/src/turn-orchestrator/provisioning/ports.ts @@ -2,15 +2,24 @@ * Typed dependency ports for provisioning. */ -import type { Model } from '../../models-catalog/types.js'; +import type { Model } from '../../types/model.js'; import type { ISdk } from '../../runtime/iii.js'; -import { getCatalogModel } from '../../runtime/output-tokens.js'; +import { logger } from '../../runtime/otel.js'; +import { getCatalogModel } from '../../runtime/catalog.js'; import type { RunRequest } from '../run-request.js'; import { createTurnStore } from '../state-runtime/store.js'; +const ROUTE_TIMEOUT_MS = 5_000; + export type ProvisioningPorts = { loadRunRequest(session_id: string): Promise; saveRunRequest(session_id: string, request: RunRequest): Promise; + /** + * Preview the routing decision via `router::route`; null when the router is + * unreachable or nothing routes (the chat call then omits `provider` and + * the router re-decides server-side — failing loudly if it's still down). + */ + route(provider: string, model: string): Promise; /** Resolve the full catalog entry for (provider, model); null on miss/error. */ resolveModel(provider: string, model: string): Promise; }; @@ -27,6 +36,24 @@ export function createProvisioningPorts(iii: ISdk): ProvisioningPorts { return store.saveRunRequest(session_id, request); }, + async route(provider, model) { + try { + const res = await iii.trigger({ + function_id: 'router::route', + payload: { model, ...(provider ? { provider } : {}) }, + timeoutMs: ROUTE_TIMEOUT_MS, + }); + return typeof res?.provider === 'string' && res.provider.length > 0 ? res.provider : null; + } catch (err) { + logger.warn('provisioning: router::route failed', { + provider, + model, + err: String(err), + }); + return null; + } + }, + resolveModel(provider, model) { return getCatalogModel(iii, provider, model); }, diff --git a/harness/src/turn-orchestrator/provisioning/process.ts b/harness/src/turn-orchestrator/provisioning/process.ts index 09fcdc7b6..ab288832e 100644 --- a/harness/src/turn-orchestrator/provisioning/process.ts +++ b/harness/src/turn-orchestrator/provisioning/process.ts @@ -2,10 +2,9 @@ * Load run request, build the provisioned RunRequest, and register the FSM step. */ -import type { Model } from '../../models-catalog/types.js'; +import type { Model } from '../../types/model.js'; import type { ISdk } from '../../runtime/iii.js'; import { agentTriggerTool } from '../agent-trigger.js'; -import { decide } from '../provider-router.js'; import { runTransition } from '../run-transition.js'; import type { RunRequest } from '../run-request.js'; import { TurnStepPayloadSchema, type TurnStepPayload } from '../schemas.js'; @@ -26,23 +25,27 @@ export async function processProvisioning( ): Promise { const request = await ports.loadRunRequest(rec.session_id); + // The router is the single routing authority: one `router::route` preview + // serves both prompt-family selection and model-metadata resolution, and + // the routed provider is pinned on the run request so the chat call + // executes on exactly the previewed provider. + const routed = request.model ? await ports.route(request.provider, request.model) : null; + const override = request.system_prompt.length > 0 ? request.system_prompt : null; const prompt = buildSystemPrompt({ override, mode: request.mode, - provider: request.provider, - model: request.model, + provider: routed ?? '', }); - const decision = decide({ provider: request.provider, model: request.model }); - const model_meta = request.model - ? await ports.resolveModel(decision.provider, request.model) - : null; + const model_meta = + request.model && routed ? await ports.resolveModel(routed, request.model) : null; return { kind: 'ready', runRequest: { ...request, + routed_provider: routed ?? '', system_prompt: prompt, function_schemas: [agentTriggerTool()], }, diff --git a/harness/src/turn-orchestrator/run-abort.ts b/harness/src/turn-orchestrator/run-abort.ts index a386b16d8..b08d1c7ea 100644 --- a/harness/src/turn-orchestrator/run-abort.ts +++ b/harness/src/turn-orchestrator/run-abort.ts @@ -40,6 +40,23 @@ export async function execute(iii: ISdk, payload: RunAbortPayload): Promise { + logger.debug('run::abort: router::abort failed (best-effort)', { + session_id, + err: String(err), + }); + }); + const ports = createTurnStatePorts(iii, store); const msg = syntheticAssistant({ stop_reason: 'aborted', text: 'run aborted by user' }); rec.last_assistant = msg; diff --git a/harness/src/turn-orchestrator/run-request.ts b/harness/src/turn-orchestrator/run-request.ts index 225b46600..f078f7454 100644 --- a/harness/src/turn-orchestrator/run-request.ts +++ b/harness/src/turn-orchestrator/run-request.ts @@ -14,6 +14,12 @@ export type RunRequest = { function_schemas: unknown[]; /** Optional reasoning/thinking level ('off'|'minimal'|'low'|'medium'|'high'|'xhigh'). */ thinking_level?: string; + /** + * Provider resolved by `router::route` during provisioning, pinned as the + * explicit provider on `router::chat` so preview and execution can never + * diverge. Empty when routing failed (the router re-decides server-side). + */ + routed_provider?: string; }; /** Empty run request used as the absent-record fallback in `loadRunRequest`. */ diff --git a/harness/src/turn-orchestrator/schemas.ts b/harness/src/turn-orchestrator/schemas.ts index afe9ee660..764dd7fa9 100644 --- a/harness/src/turn-orchestrator/schemas.ts +++ b/harness/src/turn-orchestrator/schemas.ts @@ -29,6 +29,12 @@ export const RunStartPayloadSchema = SessionIdPayloadSchema.extend({ provider: z.string(), model: z.string(), mode: z.enum(['plan', 'ask', 'agent'] satisfies [Mode, Mode, Mode]).optional(), + /** + * Optional reasoning/thinking level, persisted on the run request and + * threaded to `router::chat` (omitted on the wire when 'off' or absent). + * The provider degrades-with-warning when the model can't honor it. + */ + thinking_level: z.enum(['off', 'minimal', 'low', 'medium', 'high', 'xhigh']).optional(), messages: z.custom((v) => Array.isArray(v)).default([]), max_turns: z.number().optional(), system_prompt: z.string().default(''), diff --git a/harness/src/turn-orchestrator/state.ts b/harness/src/turn-orchestrator/state.ts index f8cf83b96..ef69ab304 100644 --- a/harness/src/turn-orchestrator/state.ts +++ b/harness/src/turn-orchestrator/state.ts @@ -7,7 +7,7 @@ */ import { z } from 'zod'; -import type { Model } from '../models-catalog/types.js'; +import type { Model } from '../types/model.js'; import type { AssistantMessage, FunctionResultMessage } from '../types/agent-message.js'; import type { ExecutedCall, FunctionBatchWork, PreparedCall } from './function-execute/types.js'; diff --git a/harness/src/turn-orchestrator/system-prompt.ts b/harness/src/turn-orchestrator/system-prompt.ts index 6c8c052d7..cd168b023 100644 --- a/harness/src/turn-orchestrator/system-prompt.ts +++ b/harness/src/turn-orchestrator/system-prompt.ts @@ -30,15 +30,13 @@ export type SystemPromptOptions = { override?: string | null; /** Operating mode; prepends a mode paragraph before the identity prompt. */ mode?: Mode | null; - /** Run's provider id (e.g. `anthropic`); selects the prompt family. */ + /** Run's ROUTED provider id (from `router::route`); selects the prompt family. */ provider?: string | null; - /** Run's model id (e.g. `gpt-5`); refines the family when provider is empty. */ - model?: string | null; }; export function buildSystemPrompt(opts: SystemPromptOptions = {}): string { - const { override, mode, provider, model } = opts; + const { override, mode, provider } = opts; if (override && override.length > 0) return override; - const identity = selectIdentityPrompt(provider ?? '', model ?? ''); + const identity = selectIdentityPrompt(provider ?? ''); return isMode(mode) ? `${MODE_PARAGRAPHS[mode]}\n\n${identity}` : identity; } diff --git a/harness/src/types/function.ts b/harness/src/types/function.ts index 55336a742..f44c7028b 100644 --- a/harness/src/types/function.ts +++ b/harness/src/types/function.ts @@ -3,6 +3,7 @@ * `harness/crates/harness-types/src/function.rs`. */ +import { z } from 'zod'; import type { ContentBlock } from './content.js'; export type ExecutionMode = 'parallel' | 'sequential'; @@ -26,6 +27,19 @@ export type AgentFunction = { prepare_arguments_supported?: boolean; }; +/** + * Zod schema for {@link AgentFunction}, used to validate the run request's + * stored function schemas before they ride to the router on `router::chat`. + */ +export const AgentFunctionSchema = z.object({ + name: z.string(), + description: z.string(), + parameters: z.unknown().default({}), + label: z.string().optional(), + execution_mode: z.enum(['parallel', 'sequential']).optional(), + prepare_arguments_supported: z.boolean().optional(), +}); + /** A single function call emitted by the assistant. */ export type FunctionCall = { id: string; diff --git a/harness/src/types/index.ts b/harness/src/types/index.ts index b5d0e7248..cb9ceea1b 100644 --- a/harness/src/types/index.ts +++ b/harness/src/types/index.ts @@ -2,7 +2,6 @@ export * from './agent-event.js'; export * from './agent-message.js'; export * from './content.js'; export * from './function.js'; -export * from './provider.js'; export * from './stream-event.js'; export * from './thinking.js'; export * from './wire.js'; diff --git a/harness/src/types/model.ts b/harness/src/types/model.ts new file mode 100644 index 000000000..baaf58cb2 --- /dev/null +++ b/harness/src/types/model.ts @@ -0,0 +1,52 @@ +/** + * Catalog model types, wire-aligned with the llm-router worker + * (`llm-router/src/types/model.rs`). The router's catalog is the source of + * truth: models are written via `router::models::reconcile` and read back via + * `router::models::get`/`list`. The router only persists the fields its + * `Model` struct declares — provider-side extras (`api`, `transports`, + * `default_cache_retention`) are accepted in a reconcile payload but dropped + * on the round-trip, so every field beyond the identity/limit core is + * optional and consumers read them defensively. + */ + +export type ThinkingLevel = 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; + +export type Transport = 'sse' | 'websocket' | 'auto'; + +export type CacheRetention = 'none' | 'short' | 'long'; + +export type ThinkingBudgets = { + minimal?: number; + low?: number; + medium?: number; + high?: number; +}; + +/** Router pricing shape (per-1M-token rates; all optional). */ +export type Pricing = { + input?: number; + output?: number; + cache_read?: number; + cache_write?: number; +}; + +export type Model = { + id: string; + provider: string; + display_name?: string; + context_window: number; + max_output_tokens?: number; + input_limit?: number; + supports_thinking?: boolean; + supports_xhigh?: boolean; + supports_tools?: boolean; + supports_vision?: boolean; + supports_cache?: boolean; + supports_structured_output?: boolean; + thinking_budgets?: ThinkingBudgets; + pricing?: Pricing; + /** Provider-side hints; not persisted by the router catalog. */ + api?: string; + transports?: Transport[]; + default_cache_retention?: CacheRetention; +}; diff --git a/harness/src/types/provider.ts b/harness/src/types/provider.ts deleted file mode 100644 index d4f362e27..000000000 --- a/harness/src/types/provider.ts +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Provider streaming contract, consumed by both the orchestrator (caller) and - * provider workers (handler). - */ - -import { z } from 'zod'; -import { zodToJsonSchema } from 'zod-to-json-schema'; -import type { Model } from '../models-catalog/types.js'; - -/** - * Lax `StreamChannelRef` — the SDK's canonical type lives in `iii-sdk`, - * but we only need the wire shape here. - */ -export const StreamChannelRefSchema = z.object({ - channel_id: z.string(), - access_key: z.string(), - direction: z.enum(['read', 'write']), -}); -export type StreamChannelRef = z.infer; - -export const AgentFunctionSchema = z.object({ - name: z.string(), - description: z.string(), - parameters: z.unknown().default({}), - label: z.string().optional(), - execution_mode: z.enum(['parallel', 'sequential']).optional(), - prepare_arguments_supported: z.boolean().optional(), -}); - -export const ProviderStreamInputSchema = z.object({ - /** - * Writer end of the channel the orchestrator opened. Provider writes - * AssistantMessageEvent JSON text messages here, then closes. - */ - writer_ref: StreamChannelRefSchema, - system_prompt: z.string().nullable().optional(), - model: z.string(), - /** Pass-through; the providers serialize this themselves. */ - messages: z.array(z.unknown()), - tools: z.array(AgentFunctionSchema).default([]), - /** - * Optional reasoning/thinking level. Providers that support it map this - * onto their native parameter (Anthropic `thinking`, OpenAI - * `reasoning_effort`); others ignore it. Absent = off. - */ - thinking_level: z.string().optional(), - /** - * Optional pre-resolved catalog entry for `model`, threaded from the - * orchestrator so the provider does not re-fetch `models::get` it already - * resolved at turn start. An optimization, never a source of truth: validated - * leniently (any object passes; anything else coerces to absent) so a sparse - * or partial catalog entry falls back to a live fetch instead of failing the - * whole stream. Providers read its fields defensively. - */ - model_meta: z - .unknown() - .optional() - .transform((v) => (v && typeof v === 'object' ? (v as Model) : undefined)), - /** - * Optional stable id for the turn (the run's start time). Providers may use - * it to dedupe per-stream credential resolution within a turn; a new turn - * carries a new key. Purely an optimization key — never a source of truth. - */ - resolution_key: z.number().optional(), -}); -export type ProviderStreamInput = z.infer; - -/** - * Runtime-side schema used by the *receiving* provider handler. The - * iii-sdk auto-hydrates any value matching {@link StreamChannelRefSchema} - * (`{channel_id, access_key, direction}`) into a `ChannelWriter` / - * `ChannelReader` *instance* via `resolveChannelValue` before the - * handler runs (see iii-sdk/dist/index.mjs `resolveChannelValue`). So by - * the time the handler sees the payload, `writer_ref` is no longer the - * wire shape — it's a class instance with `sendMessage` / `close`. We - * therefore relax just that one field for runtime validation; the - * documented wire shape is still {@link ProviderStreamInputSchema}. - */ -export const ProviderStreamRuntimeInputSchema = ProviderStreamInputSchema.extend({ - writer_ref: z.unknown(), -}); - -export const ProviderStreamOutputSchema = z.object({ - ok: z.boolean(), - status: z.string().optional(), -}); -export type ProviderStreamOutput = z.infer; - -/** Auto-derived JSON schemas, exposed via `engine::functions::info`. */ -export const ProviderStreamInputJsonSchema = zodToJsonSchema(ProviderStreamInputSchema, { - name: 'ProviderStreamInput', -}); -export const ProviderStreamOutputJsonSchema = zodToJsonSchema(ProviderStreamOutputSchema, { - name: 'ProviderStreamOutput', -}); diff --git a/harness/tests/context-compaction/compact-session-registered.test.ts b/harness/tests/context-compaction/compact-session-registered.test.ts index cc794e3d4..947d44d1f 100644 --- a/harness/tests/context-compaction/compact-session-registered.test.ts +++ b/harness/tests/context-compaction/compact-session-registered.test.ts @@ -52,17 +52,6 @@ function buildSdk(opts: { const handlers = new Map(); const stateStore = new Map(); - let channelCb: ((raw: string) => void) | null = null; - const channel = { - reader: { - onMessage(cb: (raw: string) => void) { - channelCb = cb; - }, - stream: { resume: () => {} }, - }, - writerRef: 'mock-writer-ref', - }; - const trigger = vi.fn(async (req: { function_id: string; payload?: unknown }) => { const fn = req.function_id; const payload = req.payload; @@ -98,13 +87,12 @@ function buildSdk(opts: { if (fn === 'session::update_message') { return { updated: true, revision: 1 }; } - if (fn === 'models::get') { - return { context_window: 200_000, max_output_tokens: 4_096 }; + if (fn === 'router::models::get') { + return { model: { context_window: 200_000, max_output_tokens: 4_096 } }; } - if (fn.startsWith('provider::')) { + if (fn === 'router::complete') { const summary = opts.summaryText ?? 'summary text here'; - const event = JSON.stringify({ - type: 'done', + return { message: { role: 'assistant', content: [{ type: 'text', text: summary }], @@ -113,21 +101,17 @@ function buildSdk(opts: { provider: 'anthropic', timestamp: Date.now(), }, - }); - if (channelCb) channelCb(event); - return undefined; + }; } return null; }); - const createChannel = vi.fn(async () => channel); const registerFunction = vi.fn((id: string, h: Handler) => { handlers.set(id, h); }); const iii = { trigger, - createChannel, registerFunction, registerTrigger: vi.fn(), publish: vi.fn(), diff --git a/harness/tests/context-compaction/compact-session.test.ts b/harness/tests/context-compaction/compact-session.test.ts index 60627a648..311aee447 100644 --- a/harness/tests/context-compaction/compact-session.test.ts +++ b/harness/tests/context-compaction/compact-session.test.ts @@ -202,8 +202,8 @@ describe('compact_session smoke', () => { async ({ function_id, payload }: { function_id: string; payload: unknown }) => { const p = (payload ?? {}) as Record; if (function_id === 'session::messages') return { messages: [] }; - if (function_id === 'models::get') { - return { context_window: 200_000, max_output_tokens: 4_096 }; + if (function_id === 'router::models::get') { + return { model: { context_window: 200_000, max_output_tokens: 4_096 } }; } if (function_id === 'state::get') { const v = stateStore.get(p.key as string); @@ -288,10 +288,10 @@ describe('compact_session smoke', () => { limit: { context: 200_000, input: 200_000, output: 4_096 }, }); expect(['ok', 'empty', 'overflow', 'busy']).toContain(result.status); - // Confirm models::get was NOT called (limits supplied inline). + // Confirm router::models::get was NOT called (limits supplied inline). expect( (iii.trigger as ReturnType).mock.calls.filter( - (c) => (c[0] as { function_id: string }).function_id === 'models::get', + (c) => (c[0] as { function_id: string }).function_id === 'router::models::get', ).length, ).toBe(0); }); diff --git a/harness/tests/context-compaction/e2e/full-session.test.ts b/harness/tests/context-compaction/e2e/full-session.test.ts index 8901ea109..5f32507a3 100644 --- a/harness/tests/context-compaction/e2e/full-session.test.ts +++ b/harness/tests/context-compaction/e2e/full-session.test.ts @@ -83,18 +83,6 @@ function buildTestSdk(opts: { const handlers = new Map(); const sessions = new FakeSessionManager(); - // Stub channel writer so streamAndCollect can deliver a synthetic done event. - let channelCb: ((raw: string) => void) | null = null; - const channel = { - reader: { - onMessage(cb: (raw: string) => void) { - channelCb = cb; - }, - stream: { resume: () => {} }, - }, - writerRef: 'mock-writer-ref', - }; - const trigger = vi.fn(async (req: { function_id: string; payload?: unknown }) => { const fn = req.function_id; const payload = req.payload; @@ -129,20 +117,24 @@ function buildTestSdk(opts: { return { old_value: oldValue ?? null, new_value: newValue ?? null }; } - // 2) models::get — return a small-context model so the 30-turn fixture overflows. - if (fn === 'models::get') { + // 2) router::models::get — a small-context model so the 30-turn fixture overflows. + if (fn === 'router::models::get') { return { - id: MODEL_ID, - provider: PROVIDER_ID, - context_window: MODEL_LIMITS.input, - max_output_tokens: MODEL_LIMITS.output, + model: { + id: MODEL_ID, + provider: PROVIDER_ID, + context_window: MODEL_LIMITS.input, + max_output_tokens: MODEL_LIMITS.output, + }, }; } - // 3) Provider stream — emit a single 'done' event with the canned summary. - if (fn.startsWith('provider::')) { - const summaryDone = { - type: 'done', + // 3) Summariser — router::complete returns the canned summary message. + if (fn === 'router::complete') { + // Capture what the summariser was asked to look at. + const completeInput = (payload ?? {}) as { messages?: AgentMessage[] }; + if (completeInput.messages) opts.providerInvocations.push(completeInput.messages); + return { message: { role: 'assistant', content: [{ type: 'text', text: opts.summaryText }], @@ -152,11 +144,6 @@ function buildTestSdk(opts: { timestamp: Date.now(), }, }; - // Capture what the summariser was asked to look at. - const streamInput = (payload ?? {}) as { messages?: AgentMessage[] }; - if (streamInput.messages) opts.providerInvocations.push(streamInput.messages); - if (channelCb) channelCb(JSON.stringify(summaryDone)); - return undefined; } // 4) session::* — the in-memory session-manager fake. @@ -169,15 +156,12 @@ function buildTestSdk(opts: { return null; }); - const createChannel = vi.fn(async () => channel); - const registerFunction = vi.fn((id: string, handler: FunctionHandler) => { handlers.set(id, handler); }); const iii = { trigger, - createChannel, registerFunction, registerTrigger: vi.fn(), publish: vi.fn(), diff --git a/harness/tests/context-compaction/handler-async.test.ts b/harness/tests/context-compaction/handler-async.test.ts index 91ca0dc9b..23c03f872 100644 --- a/harness/tests/context-compaction/handler-async.test.ts +++ b/harness/tests/context-compaction/handler-async.test.ts @@ -52,13 +52,13 @@ describe('resolveModel', () => { const iii = { trigger: vi.fn(async (req: { function_id: string }) => { calls.push(req.function_id); - return { context_window: 200_000, max_output_tokens: 8_096 }; + return { model: { context_window: 200_000, max_output_tokens: 8_096 } }; }), } as unknown as ISdk; return { iii, calls }; } - it('uses the threaded limit and skips models::get', async () => { + it('uses the threaded limit and skips router::models::get', async () => { const { iii, calls } = trackingIii(); const resolved = await resolveModel(iii, 'sess-1', 'anthropic', 'claude-sonnet-4-6', { @@ -67,7 +67,7 @@ describe('resolveModel', () => { output: 64_000, }); - expect(calls).not.toContain('models::get'); + expect(calls).not.toContain('router::models::get'); expect(resolved).toEqual({ providerID: 'anthropic', modelID: 'claude-sonnet-4-6', @@ -75,11 +75,11 @@ describe('resolveModel', () => { }); }); - it('falls back to models::get when no threaded limit is present', async () => { + it('falls back to router::models::get when no threaded limit is present', async () => { const { iii, calls } = trackingIii(); await resolveModel(iii, 'sess-1', 'anthropic', 'claude-sonnet-4-6'); - expect(calls).toContain('models::get'); + expect(calls).toContain('router::models::get'); }); }); diff --git a/harness/tests/context-compaction/integration/backward-compat.test.ts b/harness/tests/context-compaction/integration/backward-compat.test.ts index 34fc4084a..ac05ba876 100644 --- a/harness/tests/context-compaction/integration/backward-compat.test.ts +++ b/harness/tests/context-compaction/integration/backward-compat.test.ts @@ -29,17 +29,6 @@ function buildBackwardCompatMock(opts: { const { fixtureMessages, capturedSystemPrompts, compactPayloads } = opts; const stateStore = new Map(); - let channelCb: ((raw: string) => void) | null = null; - - const channel = { - reader: { - onMessage(cb: (raw: string) => void) { - channelCb = cb; - }, - stream: { resume: () => {} }, - }, - writerRef: 'mock-writer-ref', - }; const trigger = vi.fn(async (req: MockTriggerReq) => { const { function_id, payload } = req; @@ -107,34 +96,31 @@ function buildBackwardCompatMock(opts: { } return { old_value: oldValue ?? null, new_value: newValue ?? null }; } - if (function_id.startsWith('provider::')) { + if (function_id === 'router::models::get') { + return { model: { context_window: 200_000, max_output_tokens: 4_096 } }; + } + if (function_id === 'router::complete') { // Capture the system_prompt sent to the summariser const systemPrompt = payload.system_prompt; if (typeof systemPrompt === 'string') { capturedSystemPrompts.push(systemPrompt); } - // Deliver a successful summary via the channel - if (channelCb) { - const msg: AssistantMessage = { - role: 'assistant', - content: [{ type: 'text', text: 'Updated anchored summary.' }], - stop_reason: 'end', - model: 'claude-haiku-4-5', - provider: 'anthropic', - timestamp: Date.now(), - }; - channelCb(JSON.stringify({ type: 'done', message: msg })); - } - return undefined; + const msg: AssistantMessage = { + role: 'assistant', + content: [{ type: 'text', text: 'Updated anchored summary.' }], + stop_reason: 'end', + model: 'claude-haiku-4-5', + provider: 'anthropic', + timestamp: Date.now(), + }; + return { message: msg }; } return undefined; }); - const createChannel = vi.fn(async () => channel); - - const iii = { trigger, createChannel } as unknown as import('../../../src/runtime/iii.js').ISdk; + const iii = { trigger } as unknown as import('../../../src/runtime/iii.js').ISdk; return { iii, trigger }; } diff --git a/harness/tests/context-compaction/integration/flow-async.test.ts b/harness/tests/context-compaction/integration/flow-async.test.ts index 2a391bff6..e3c3d3069 100644 --- a/harness/tests/context-compaction/integration/flow-async.test.ts +++ b/harness/tests/context-compaction/integration/flow-async.test.ts @@ -43,30 +43,15 @@ function buildAsyncMock(opts: { stateStore = new Map(), } = opts; - let channelCb: ((raw: string) => void) | null = null; - - const channel = { - reader: { - onMessage(cb: (raw: string) => void) { - channelCb = cb; - }, - stream: { resume: () => {} }, - }, - writerRef: 'mock-writer-ref', + const doneSummaryMessage = { + role: 'assistant', + content: [{ type: 'text', text: 'Async compaction summary.' }], + stop_reason: 'end', + model: 'claude-haiku-4-5', + provider: 'anthropic', + timestamp: Date.now(), }; - const doneSummaryEvent = JSON.stringify({ - type: 'done', - message: { - role: 'assistant', - content: [{ type: 'text', text: 'Async compaction summary.' }], - stop_reason: 'end', - model: 'claude-haiku-4-5', - provider: 'anthropic', - timestamp: Date.now(), - }, - }); - let appendSeq = 0; const trigger = vi.fn(async (req: MockTriggerReq) => { const { function_id, payload } = req; @@ -81,8 +66,8 @@ function buildAsyncMock(opts: { if (function_id === 'session::update_message') { return { updated: true, revision: 1 }; } - if (function_id === 'models::get') { - return modelCatalog; + if (function_id === 'router::models::get') { + return modelCatalog ? { model: modelCatalog } : null; } if (function_id === 'state::get') { const v = stateStore.get((payload as { key: string }).key); @@ -111,21 +96,16 @@ function buildAsyncMock(opts: { } return { old_value: oldValue ?? null, new_value: newValue ?? null }; } - if (function_id.startsWith('provider::')) { - if (channelCb) { - channelCb(doneSummaryEvent); - } - return undefined; + if (function_id === 'router::complete') { + return { message: doneSummaryMessage }; } return undefined; }); - const createChannel = vi.fn(async () => channel); - - const iii = { trigger, createChannel } as unknown as import('../../../src/runtime/iii.js').ISdk; + const iii = { trigger } as unknown as import('../../../src/runtime/iii.js').ISdk; - return { iii, trigger, createChannel, compactPayloads }; + return { iii, trigger, compactPayloads }; } /** diff --git a/harness/tests/context-compaction/integration/flow-sync.test.ts b/harness/tests/context-compaction/integration/flow-sync.test.ts index 64b8d9140..198714e79 100644 --- a/harness/tests/context-compaction/integration/flow-sync.test.ts +++ b/harness/tests/context-compaction/integration/flow-sync.test.ts @@ -30,7 +30,7 @@ const defaultModel = { * Build an ISdk-compatible mock for the sync flow. * * @param opts.fixtureMessages - entries returned by session::messages - * @param opts.providerError - if true, make createChannel reject to simulate summariser error + * @param opts.providerError - if true, router::complete rejects to simulate a summariser error * @param opts.stateStore - optional shared state store for lease simulation */ function buildSyncMock(opts: { @@ -40,30 +40,15 @@ function buildSyncMock(opts: { }) { const { fixtureMessages, providerError = false, stateStore = new Map() } = opts; - let channelCb: ((raw: string) => void) | null = null; - - const channel = { - reader: { - onMessage(cb: (raw: string) => void) { - channelCb = cb; - }, - stream: { resume: () => {} }, - }, - writerRef: 'mock-writer-ref', + const doneSummaryMessage = { + role: 'assistant', + content: [{ type: 'text', text: 'Sync compaction summary text.' }], + stop_reason: 'end', + model: 'claude-haiku-4-5', + provider: 'anthropic', + timestamp: Date.now(), }; - const doneSummaryEvent = JSON.stringify({ - type: 'done', - message: { - role: 'assistant', - content: [{ type: 'text', text: 'Sync compaction summary text.' }], - stop_reason: 'end', - model: 'claude-haiku-4-5', - provider: 'anthropic', - timestamp: Date.now(), - }, - }); - const compactionAppends: unknown[] = []; const messageAppends: unknown[] = []; const updateMessageCalls: unknown[] = []; @@ -117,21 +102,17 @@ function buildSyncMock(opts: { } return { old_value: oldValue ?? null, new_value: newValue ?? null }; } - if (function_id.startsWith('provider::')) { - if (channelCb) { - channelCb(doneSummaryEvent); + if (function_id === 'router::complete') { + if (providerError) { + throw new Error('summariser unavailable'); } - return undefined; + return { message: doneSummaryMessage }; } return undefined; }); - const createChannel = providerError - ? vi.fn().mockRejectedValue(new Error('provider channel unavailable')) - : vi.fn(async () => channel); - - const iii = { trigger, createChannel } as unknown as ISdk; + const iii = { trigger } as unknown as ISdk; return { iii, trigger, compactionAppends, messageAppends, updateMessageCalls }; } diff --git a/harness/tests/context-compaction/summarize.test.ts b/harness/tests/context-compaction/summarize.test.ts index 9bd405bba..cf3c8cb9b 100644 --- a/harness/tests/context-compaction/summarize.test.ts +++ b/harness/tests/context-compaction/summarize.test.ts @@ -49,20 +49,15 @@ const testModel = { }; // --------------------------------------------------------------------------- -// Channel + ISdk mock factory +// ISdk mock factory // -// streamAndCollect flow: -// 1. channel = await iii.createChannel() -// 2. channel.reader.onMessage(cb) ← registers the callback -// 3. await iii.trigger(provider...) ← we fire cb here, inside the trigger mock -// 4. while(!terminal) drain loop -// -// We fire the channel message synchronously inside the provider trigger -// call so that `terminal` is set before the drain loop runs. +// The summariser now runs through one `router::complete` call — no channel. +// `onComplete` receives the payload and returns the final AssistantMessage +// (or throws to fail the call, feeding the retry logic). // --------------------------------------------------------------------------- -function makeDoneEvent(summary = 'the summary'): string { - const msg: AssistantMessage = { +function doneMessage(summary = 'the summary'): AssistantMessage { + return { role: 'assistant', content: [{ type: 'text', text: summary }], stop_reason: 'end', @@ -70,19 +65,6 @@ function makeDoneEvent(summary = 'the summary'): string { provider: 'p', timestamp: 0, }; - return JSON.stringify({ type: 'done', message: msg }); -} - -function makeEmptyDoneEvent(): string { - const msg: AssistantMessage = { - role: 'assistant', - content: [], - stop_reason: 'end', - model: 'm', - provider: 'p', - timestamp: 0, - }; - return JSON.stringify({ type: 'done', message: msg }); } type MockTriggerReq = { function_id: string; payload: Record }; @@ -90,27 +72,14 @@ type MockTriggerReq = { function_id: string; payload: Record }; /** * Build a minimal ISdk mock. * - * @param onProviderTrigger - Called synchronously when a provider trigger fires. - * Return the raw JSON event string to deliver on the channel, or null to skip. + * @param onComplete - Called with the `router::complete` payload; returns the + * AssistantMessage wrapped as `{ message }`, or throws to fail the call. * @param extraHandlers - per function_id response overrides. */ function buildMock( - onProviderTrigger: (payload: Record) => string | null = () => makeDoneEvent(), + onComplete: (payload: Record) => AssistantMessage = () => doneMessage(), extraHandlers: Record) => unknown> = {}, ) { - // The channel message callback registered by streamAndCollect - let channelCb: ((raw: string) => void) | null = null; - - const channel = { - reader: { - onMessage(cb: (raw: string) => void) { - channelCb = cb; - }, - stream: { resume: () => {} }, - }, - writerRef: 'mock-writer-ref', - }; - const trigger = vi.fn(async (req: MockTriggerReq) => { const { function_id, payload } = req; @@ -133,25 +102,16 @@ function buildMock( return undefined; } - // Provider stream: fire channel callback synchronously so terminal is set - if (function_id.startsWith('provider::')) { - if (channelCb) { - const event = onProviderTrigger(payload); - if (event !== null) { - channelCb(event); - } - } - return undefined; + if (function_id === 'router::complete') { + return { message: onComplete(payload), provider: payload.provider, model: payload.model }; } return undefined; }); - const createChannel = vi.fn(async () => channel); - - const iii = { trigger, createChannel } as unknown as import('iii-sdk').ISdk; + const iii = { trigger } as unknown as import('iii-sdk').ISdk; - return { iii, trigger, createChannel }; + return { iii, trigger }; } // --------------------------------------------------------------------------- @@ -231,7 +191,7 @@ describe('summarizeAndAppend async mode', () => { it('loads messages, calls summariser, appends compaction, returns ok', async () => { const compactPayloads: unknown[] = []; - const { iii, trigger } = buildMock(() => makeDoneEvent('my summary'), { + const { iii, trigger } = buildMock(() => doneMessage('my summary'), { 'session::messages': () => ({ messages: makeEntries().map((e) => ({ entry_id: e.entry_id, message: e.message })), }), @@ -249,11 +209,9 @@ describe('summarizeAndAppend async mode', () => { const ok = result as { tail_start_id: string | null; tokens_before: number }; expect(ok.tokens_before).toBeGreaterThan(0); - // Provider trigger fired - const providerCall = trigger.mock.calls.find(([req]) => - req.function_id.startsWith('provider::'), - ); - expect(providerCall).toBeDefined(); + // The summariser ran through router::complete + const completeCall = trigger.mock.calls.find(([req]) => req.function_id === 'router::complete'); + expect(completeCall).toBeDefined(); // Compaction custom entry was appended with session_id and non-empty summary expect(compactPayloads).toHaveLength(1); @@ -268,62 +226,52 @@ describe('summarizeAndAppend async mode', () => { }); it('returns "empty" when summariser produces no text content', async () => { - const { iii } = buildMock(() => makeEmptyDoneEvent()); + const { iii } = buildMock(() => ({ ...doneMessage(), content: [] })); const result = await summarizeAndAppend(iii, 'sess-empty', { mode: 'async' }, testModel); expect(result).toBe('empty'); }); - it('returns "compact" when streamAndCollect throws', async () => { - const { iii } = buildMock(); - // Make createChannel reject so streamAndCollect throws - (iii as unknown as { createChannel: ReturnType }).createChannel = vi - .fn() - .mockRejectedValue(new Error('channel unavailable')); + it('returns "compact" when router::complete rejects', async () => { + const { iii } = buildMock(() => { + throw new Error('router unreachable'); + }); const result = await summarizeAndAppend(iii, 'sess-fail', { mode: 'async' }, testModel); expect(typeof result === 'object' && 'kind' in result ? result.kind : result).toBe('compact'); if (typeof result === 'object' && 'kind' in result && result.kind === 'compact') { - expect(result.reason).toContain('channel unavailable'); + expect(result.reason).toContain('router unreachable'); } }); - it('retries streamAndCollect once and succeeds when the first attempt fails', async () => { + it('retries router::complete once and succeeds when the first attempt fails', async () => { // Transient provider failures (429, network blip, 5xx) should not // surface to /compact when a single retry would succeed. - const { iii } = buildMock(); - let createChannelCalls = 0; - const realCreateChannel = (iii as unknown as { createChannel: () => Promise }) - .createChannel; - (iii as unknown as { createChannel: () => Promise }).createChannel = vi.fn( - async () => { - createChannelCalls += 1; - if (createChannelCalls === 1) { - throw new Error('transient provider failure'); - } - return realCreateChannel(); - }, - ); + let attempts = 0; + const { iii } = buildMock(() => { + attempts += 1; + if (attempts === 1) { + throw new Error('transient provider failure'); + } + return doneMessage('recovered summary'); + }); const result = await summarizeAndAppend(iii, 'sess-retry', { mode: 'async' }, testModel); - expect(createChannelCalls).toBe(2); + expect(attempts).toBe(2); expect(typeof result === 'object' && 'kind' in result ? result.kind : result).toBe('ok'); }, 10_000); it('returns "compact" when both stream attempts fail', async () => { // Generic permanent failure (no auth/4xx markers) — still retries once // then surfaces the failure. - const { iii } = buildMock(); - let createChannelCalls = 0; - (iii as unknown as { createChannel: () => Promise }).createChannel = vi.fn( - async () => { - createChannelCalls += 1; - throw new Error('permanent provider failure'); - }, - ); + let attempts = 0; + const { iii } = buildMock(() => { + attempts += 1; + throw new Error('permanent provider failure'); + }); const result = await summarizeAndAppend(iii, 'sess-fail-twice', { mode: 'async' }, testModel); - expect(createChannelCalls).toBe(2); + expect(attempts).toBe(2); expect(typeof result === 'object' && 'kind' in result ? result.kind : result).toBe('compact'); if (typeof result === 'object' && 'kind' in result && result.kind === 'compact') { expect(result.reason).toContain('permanent provider failure'); @@ -333,17 +281,14 @@ describe('summarizeAndAppend async mode', () => { it('skips retry on non-retryable errors (401/auth/malformed)', async () => { // Auth and 4xx errors won't fix on retry. Skip the retry so the // compaction lease releases ~1s sooner. - const { iii } = buildMock(); - let createChannelCalls = 0; - (iii as unknown as { createChannel: () => Promise }).createChannel = vi.fn( - async () => { - createChannelCalls += 1; - throw new Error('401 unauthorized: invalid_api_key'); - }, - ); + let attempts = 0; + const { iii } = buildMock(() => { + attempts += 1; + throw new Error('401 unauthorized: invalid_api_key'); + }); const result = await summarizeAndAppend(iii, 'sess-auth-fail', { mode: 'async' }, testModel); - expect(createChannelCalls).toBe(1); // no retry + expect(attempts).toBe(1); // no retry expect(typeof result === 'object' && 'kind' in result ? result.kind : result).toBe('compact'); if (typeof result === 'object' && 'kind' in result && result.kind === 'compact') { expect(result.reason).toContain('401'); @@ -355,25 +300,20 @@ describe('summarizeAndAppend async mode', () => { // combo) used to be silently stored as the compaction summary text, // showing "COMPACTED · N TOKENS" in the UI with the error JSON as // the summary. The error terminal must surface as a failure instead. - const errorEvent = JSON.stringify({ - type: 'error', - error: { - role: 'assistant', - content: [ - { - type: 'text', - text: '{"type":"error","error":{"type":"not_found_error","message":"model: gpt-5-mini"}}', - }, - ], - stop_reason: 'end', - error_kind: 'NotFound', - error_message: 'model: gpt-5-mini', - model: 'gpt-5-mini', - provider: 'anthropic', - timestamp: 0, - }, - }); - const { iii } = buildMock(() => errorEvent); + const { iii } = buildMock(() => ({ + role: 'assistant', + content: [ + { + type: 'text', + text: '{"type":"error","error":{"type":"not_found_error","message":"model: gpt-5-mini"}}', + }, + ], + stop_reason: 'error', + error_message: 'model: gpt-5-mini', + model: 'gpt-5-mini', + provider: 'anthropic', + timestamp: 0, + })); const result = await summarizeAndAppend( iii, @@ -387,20 +327,10 @@ describe('summarizeAndAppend async mode', () => { } }); - it('routes the summariser to the session provider', async () => { - // /compact uses the session's own provider/model. Earlier default was - // a hardcoded 'anthropic'; OpenAI sessions got an Anthropic stream - // + an OpenAI model id, producing not_found_error. - let calledFunctionId: string | null = null; - const { iii } = buildMock(); - const wrapped = iii as unknown as { trigger: ReturnType }; - const originalTrigger = wrapped.trigger; - wrapped.trigger = vi.fn(async (req: MockTriggerReq) => { - if (req.function_id.startsWith('provider::')) { - calledFunctionId = req.function_id; - } - return originalTrigger(req); - }); + it('pins the session provider/model on the router::complete call', async () => { + // /compact uses the session's own provider/model, pinned explicitly so + // the router executes on exactly the provider the session streams on. + const { iii, trigger } = buildMock(); const openAiModel = { providerID: 'openai', @@ -408,24 +338,14 @@ describe('summarizeAndAppend async mode', () => { modelLimit: { context: 200_000, input: 200_000, output: 4_096 }, }; await summarizeAndAppend(iii, 'sess-openai', { mode: 'async' }, openAiModel); - expect(calledFunctionId).toBe('provider::openai::stream'); + const call = trigger.mock.calls.find(([req]) => req.function_id === 'router::complete'); + expect(call?.[0].payload).toEqual( + expect.objectContaining({ provider: 'openai', model: 'gpt-5-mini' }), + ); }); - it('routes kimi sessions to provider::kimi::stream, not anthropic', async () => { - // Regression: a binary openai-vs-anthropic ternary sent every non-openai - // provider (including kimi) to provider::anthropic::stream, producing - // NOT_FOUND_ERROR on the kimi model id. Fix uses the canonical provider - // router so adding a provider only needs a router update. - let calledFunctionId: string | null = null; - const { iii } = buildMock(); - const wrapped = iii as unknown as { trigger: ReturnType }; - const originalTrigger = wrapped.trigger; - wrapped.trigger = vi.fn(async (req: MockTriggerReq) => { - if (req.function_id.startsWith('provider::')) { - calledFunctionId = req.function_id; - } - return originalTrigger(req); - }); + it('pins kimi sessions to the kimi provider, not anthropic', async () => { + const { iii, trigger } = buildMock(); const kimiModel = { providerID: 'kimi', @@ -433,7 +353,10 @@ describe('summarizeAndAppend async mode', () => { modelLimit: { context: 256_000, input: 256_000, output: 16_384 }, }; await summarizeAndAppend(iii, 'sess-kimi', { mode: 'async' }, kimiModel); - expect(calledFunctionId).toBe('provider::kimi::stream'); + const call = trigger.mock.calls.find(([req]) => req.function_id === 'router::complete'); + expect(call?.[0].payload).toEqual( + expect.objectContaining({ provider: 'kimi', model: 'kimi-k2.5' }), + ); }); }); @@ -449,7 +372,7 @@ describe('summarizeAndAppend anchored prompt', () => { const { iii } = buildMock( (payload) => { capturedPrompts.push(payload.system_prompt as string); - return makeDoneEvent('updated summary'); + return doneMessage('updated summary'); }, { 'session::messages': (p) => ({ @@ -495,7 +418,7 @@ describe('summarizeAndAppend sync mode', () => { const messageReads: Array> = []; const compactPayloads: unknown[] = []; - const { iii } = buildMock(() => makeDoneEvent('sync summary'), { + const { iii } = buildMock(() => doneMessage('sync summary'), { 'session::messages': (p) => { messageReads.push(p); return { messages: [] }; diff --git a/harness/tests/harness/migrate-llm-router-config.test.ts b/harness/tests/harness/migrate-llm-router-config.test.ts new file mode 100644 index 000000000..c70639a08 --- /dev/null +++ b/harness/tests/harness/migrate-llm-router-config.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + composeMigratedValue, + migrateLlmRouterConfig, + ROUTING_PARITY_SEED, +} from '../../src/harness/migrate-llm-router-config.js'; +import type { ISdk } from '../../src/runtime/iii.js'; + +type TriggerReq = { function_id: string; payload: Record }; + +function makeIii(opts: { + marker?: unknown; + routerValue?: unknown; + harnessValue?: unknown; + failSets?: number; +}) { + const sets: unknown[] = []; + const markerWrites: unknown[] = []; + let setFailures = opts.failSets ?? 0; + const trigger = vi.fn(async (req: TriggerReq) => { + const { function_id, payload } = req; + if (function_id === 'state::get') { + return opts.marker ?? null; + } + if (function_id === 'state::set') { + markerWrites.push(payload.value); + return { ok: true }; + } + if (function_id === 'configuration::get') { + if (payload.id === 'llm-router') return { id: 'llm-router', value: opts.routerValue ?? null }; + if (payload.id === 'harness') return { id: 'harness', value: opts.harnessValue ?? null }; + return null; + } + if (function_id === 'configuration::set') { + if (setFailures > 0) { + setFailures -= 1; + throw new Error('schema validation failed: unknown entry'); + } + sets.push(payload); + return { ok: true }; + } + return null; + }); + const iii = { trigger } as unknown as ISdk; + return { iii, trigger, sets, markerWrites }; +} + +describe('composeMigratedValue', () => { + it('copies providers and seeds routing parity', () => { + const out = composeMigratedValue(null, { anthropic: { api_key: 'sk-1' } }); + expect(out.providers).toEqual({ anthropic: { api_key: 'sk-1' } }); + expect(out.default_provider).toBe(ROUTING_PARITY_SEED.default_provider); + expect(out.routing_heuristics).toEqual(ROUTING_PARITY_SEED.routing_heuristics); + }); + + it('never clobbers operator-set routing or provider slices', () => { + const out = composeMigratedValue( + { + default_provider: 'openai', + routing_heuristics: [{ pattern: '^x-', provider: 'kimi' }], + providers: { anthropic: { api_key: 'operator-key' } }, + }, + { anthropic: { api_key: 'legacy-key' }, openai: { api_key: 'sk-o' } }, + ); + expect(out.default_provider).toBe('openai'); + expect(out.routing_heuristics).toEqual([{ pattern: '^x-', provider: 'kimi' }]); + // Operator slice wins over the legacy copy; missing slices are added. + expect(out.providers).toEqual({ + anthropic: { api_key: 'operator-key' }, + openai: { api_key: 'sk-o' }, + }); + }); +}); + +describe('migrateLlmRouterConfig', () => { + it('no-ops when the marker is already set', async () => { + const { iii, sets, trigger } = makeIii({ marker: { migrated_at: '2026-06-01' } }); + await migrateLlmRouterConfig(iii); + expect(sets).toHaveLength(0); + expect( + trigger.mock.calls.some(([req]) => (req as TriggerReq).function_id === 'configuration::set'), + ).toBe(false); + }); + + it('treats an operator-populated llm-router entry as migrated', async () => { + const { iii, sets, markerWrites } = makeIii({ + routerValue: { providers: { anthropic: { api_key: 'sk-op' } } }, + }); + await migrateLlmRouterConfig(iii); + expect(sets).toHaveLength(0); + expect(markerWrites).toHaveLength(1); + }); + + it('copies the harness providers block, seeds parity, and writes the marker', async () => { + const { iii, sets, markerWrites } = makeIii({ + harnessValue: { + permissions: { default_mode: 'manual' }, + providers: { anthropic: { api_key: '${ANTHROPIC_API_KEY:}' } }, + }, + }); + await migrateLlmRouterConfig(iii); + + expect(sets).toHaveLength(1); + const set = sets[0] as { id: string; value: Record }; + expect(set.id).toBe('llm-router'); + // Raw template form survives the copy verbatim. + expect(set.value.providers).toEqual({ anthropic: { api_key: '${ANTHROPIC_API_KEY:}' } }); + expect(set.value.default_provider).toBe('anthropic'); + expect(markerWrites).toHaveLength(1); + }); + + it('retries the set while the entry schema is still composing', async () => { + vi.useFakeTimers(); + try { + const { iii, sets, markerWrites } = makeIii({ + harnessValue: { providers: { anthropic: { api_key: 'sk-1' } } }, + failSets: 1, + }); + const run = migrateLlmRouterConfig(iii); + await vi.advanceTimersByTimeAsync(5_000); + await run; + expect(sets).toHaveLength(1); + expect(markerWrites).toHaveLength(1); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/harness/tests/harness/policy.test.ts b/harness/tests/harness/policy.test.ts index 0d7affc4e..d5c752ede 100644 --- a/harness/tests/harness/policy.test.ts +++ b/harness/tests/harness/policy.test.ts @@ -558,13 +558,18 @@ describe('shipped iii-permissions.yaml', () => { 'state::delete', 'stream::set', 'iii::durable::publish', - 'harness::provider::resolve', - 'harness::provider::register', + 'router::provider::resolve', + 'router::provider::register', + 'router::provider::update_credential', + 'router::models::reconcile', + 'router::chat', + 'router::complete', + 'router::abort', + 'router::route', 'configuration::get', 'configuration::set', 'configuration::register', 'run::start', - 'router::stream_assistant', ]; it('kernel surfaces are denied unconditionally — hostile args cannot dodge them', async () => { @@ -586,7 +591,14 @@ describe('shipped iii-permissions.yaml', () => { it('allows the read-only conveniences', async () => { const perms = await load(); - for (const fid of ['state::get', 'state::list', 'models::list', 'models::get']) { + for (const fid of [ + 'state::get', + 'state::list', + 'router::models::list', + 'router::models::get', + 'router::models::supports', + 'router::provider::list', + ]) { expect(perms.check(fid, {}).kind).toBe('allow'); } }); diff --git a/harness/tests/harness/providers/refresh-on-config.test.ts b/harness/tests/harness/providers/refresh-on-config.test.ts deleted file mode 100644 index 86ae5183c..000000000 --- a/harness/tests/harness/providers/refresh-on-config.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - normalizeHarnessConfig, - providerDiscoveryFingerprint, - providersAffectedByConfigChange, -} from '../../../src/runtime/harness-config.js'; - -describe('providersAffectedByConfigChange', () => { - it('returns [] when only permissions change', () => { - const oldCfg = normalizeHarnessConfig({ - permissions: { default_mode: 'manual' }, - providers: { anthropic: { api_key: 'sk-a' } }, - }); - const newCfg = normalizeHarnessConfig({ - permissions: { default_mode: 'auto' }, - providers: { anthropic: { api_key: 'sk-a' } }, - }); - expect(providersAffectedByConfigChange(oldCfg, newCfg)).toEqual([]); - }); - - it('returns anthropic when its api_key changes', () => { - const oldCfg = normalizeHarnessConfig({ - permissions: { default_mode: 'manual' }, - providers: { - anthropic: { api_key: 'sk-old' }, - openai: { api_key: 'sk-o' }, - }, - }); - const newCfg = normalizeHarnessConfig({ - permissions: { default_mode: 'manual' }, - providers: { - anthropic: { api_key: 'sk-new' }, - openai: { api_key: 'sk-o' }, - }, - }); - expect(providersAffectedByConfigChange(oldCfg, newCfg)).toEqual(['anthropic']); - }); - - it('treats empty and missing api_key consistently', () => { - const withEmpty = providerDiscoveryFingerprint({ api_key: '' }); - const missing = providerDiscoveryFingerprint({}); - expect(withEmpty).toBe(missing); - expect(withEmpty).toContain('api_key='); - }); - - it('detects api_url changes', () => { - const oldCfg = normalizeHarnessConfig({ - permissions: { default_mode: 'manual' }, - providers: { lmstudio: { api_url: 'http://localhost:1234/v1/chat/completions' } }, - }); - const newCfg = normalizeHarnessConfig({ - permissions: { default_mode: 'manual' }, - providers: { lmstudio: { api_url: 'http://localhost:5678/v1/chat/completions' } }, - }); - expect(providersAffectedByConfigChange(oldCfg, newCfg)).toEqual(['lmstudio']); - }); -}); diff --git a/harness/tests/harness/providers/registry.test.ts b/harness/tests/harness/providers/registry.test.ts deleted file mode 100644 index 89a1db6ea..000000000 --- a/harness/tests/harness/providers/registry.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { ProviderRegistry } from '../../../src/harness/providers/registry.js'; -import type { ISdk } from '../../../src/runtime/iii.js'; - -/** - * In-memory stand-in for the `configuration` worker: keeps one value + the - * last-registered schema so we can assert the dynamic schema composition and - * the resolve path without a live engine. - */ -function makeConfigSdk(initialValue: unknown) { - const state: { value: unknown; schema: Record | null } = { - value: initialValue, - schema: null, - }; - const trigger = vi.fn(async (req: { function_id: string; payload: Record }) => { - switch (req.function_id) { - case 'configuration::register': { - state.schema = req.payload.schema as Record; - if (state.value === null && req.payload.initial_value !== undefined) { - state.value = req.payload.initial_value; - } - return {}; - } - case 'configuration::get': - return { id: req.payload.id, value: state.value }; - case 'configuration::set': - state.value = req.payload.value; - return { new_value: state.value, old_value: null }; - default: - return null; - } - }); - const sdk = { trigger, registerFunction: vi.fn() } as unknown as ISdk; - return { sdk, state }; -} - -describe('ProviderRegistry.init', () => { - it('seeds the base harness entry (permissions + empty providers)', async () => { - const { sdk, state } = makeConfigSdk(null); - await new ProviderRegistry(sdk).init(); - expect(state.value).toEqual({ - permissions: { default_mode: 'manual' }, - providers: {}, - }); - }); -}); - -describe('ProviderRegistry.declare', () => { - it('composes a dynamic schema with one property per declared provider', async () => { - const { sdk, state } = makeConfigSdk(null); - const reg = new ProviderRegistry(sdk); - await reg.declare({ - id: 'anthropic', - defaults: { api_url: 'https://api.anthropic.com/v1/messages', max_tokens: 8192 }, - }); - await reg.declare({ id: 'openai', defaults: { max_tokens: 8192 } }); - - const schema = state.schema as Record>>; - const providers = schema.properties.providers as Record>; - expect(Object.keys(providers.properties)).toEqual(['anthropic', 'openai']); - const permissions = schema.properties.permissions as Record>; - const mode = (permissions.properties as Record>).default_mode; - expect(mode.enum).toEqual(['manual', 'auto', 'full']); - }); -}); - -describe('ProviderRegistry.resolve', () => { - it('returns the stored api key + settings (source=stored)', async () => { - const { sdk } = makeConfigSdk({ - permissions: { default_mode: 'auto' }, - providers: { - anthropic: { api_key: 'sk-abc', api_url: 'https://custom', max_tokens: 5000 }, - }, - }); - const reg = new ProviderRegistry(sdk); - await reg.declare({ - id: 'anthropic', - credential_env_var: 'ANTHROPIC_API_KEY', - defaults: { api_url: 'https://default', max_tokens: 8192 }, - }); - const r = await reg.resolve('anthropic'); - expect(r.credential).toEqual({ type: 'api_key', key: 'sk-abc' }); - expect(r.source).toBe('stored'); - expect(r.api_url).toBe('https://custom'); - expect(r.max_tokens).toBe(5000); - expect(r.configured).toBe(true); - }); - - it('falls back to the env var and declared defaults (source=environment)', async () => { - process.env.III_TEST_PROVIDER_KEY = 'sk-env'; - try { - const { sdk } = makeConfigSdk({ permissions: { default_mode: 'manual' }, providers: {} }); - const reg = new ProviderRegistry(sdk); - await reg.declare({ - id: 'foo', - credential_env_var: 'III_TEST_PROVIDER_KEY', - defaults: { api_url: 'https://default', max_tokens: 1234 }, - }); - const r = await reg.resolve('foo'); - expect(r.credential).toEqual({ type: 'api_key', key: 'sk-env' }); - expect(r.source).toBe('environment'); - expect(r.api_url).toBe('https://default'); - // The declared default max_tokens is NOT seeded as an override — - // seeding it would pin every request to 8192 and defeat the clamp. - expect(r.max_tokens).toBeNull(); - } finally { - process.env.III_TEST_PROVIDER_KEY = undefined; - } - }); - - it('returns max_tokens only when the user actually configured it', async () => { - const { sdk } = makeConfigSdk({ - permissions: { default_mode: 'manual' }, - providers: { anthropic: { api_key: 'sk-abc' } }, - }); - const reg = new ProviderRegistry(sdk); - await reg.declare({ - id: 'anthropic', - credential_env_var: 'ANTHROPIC_API_KEY', - defaults: { api_url: 'https://default', max_tokens: 8192 }, - }); - const r = await reg.resolve('anthropic'); - expect(r.max_tokens).toBeNull(); - }); - - it('reports unconfigured when neither stored nor env credential exists', async () => { - const { sdk } = makeConfigSdk({ permissions: { default_mode: 'manual' }, providers: {} }); - const reg = new ProviderRegistry(sdk); - await reg.declare({ id: 'bar', credential_env_var: 'III_TEST_UNSET_KEY', defaults: {} }); - const r = await reg.resolve('bar'); - expect(r.credential).toBeNull(); - expect(r.source).toBeNull(); - expect(r.configured).toBe(false); - }); -}); diff --git a/harness/tests/models-catalog/state.test.ts b/harness/tests/models-catalog/state.test.ts deleted file mode 100644 index 6aab9e063..000000000 --- a/harness/tests/models-catalog/state.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { isModel, providerStateKey } from '../../src/models-catalog/state.js'; - -describe('provider catalog state helpers', () => { - it('providerStateKey uses bare provider id', () => { - expect(providerStateKey('anthropic')).toBe('anthropic'); - }); - - it('isModel guards the write-side boundary (object with a string id)', () => { - const model = { - id: 'm1', - provider: 'anthropic', - api: 'anthropic-messages', - display_name: 'm1', - context_window: 1, - }; - expect(isModel(model)).toBe(true); - expect(isModel({ provider: 'anthropic' })).toBe(false); - expect(isModel(null)).toBe(false); - expect(isModel('bad')).toBe(false); - }); -}); diff --git a/harness/tests/models-catalog/types.test.ts b/harness/tests/models-catalog/types.test.ts deleted file mode 100644 index 26ad3ef6a..000000000 --- a/harness/tests/models-catalog/types.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { type Model, parseCapability, supportsModel } from '../../src/models-catalog/types.js'; - -const m = (overrides: Partial): Model => ({ - id: 'm', - provider: 'p', - api: 'a', - display_name: 'M', - context_window: 100, - ...overrides, -}); - -describe('parseCapability', () => { - it('parses known capability strings', () => { - expect(parseCapability('thinking')).toEqual({ type: 'thinking' }); - expect(parseCapability('thinking:xhigh')).toEqual({ - type: 'thinking_level', - level: 'xhigh', - }); - expect(parseCapability('tools')).toEqual({ type: 'tools' }); - }); - - it('returns null for unknown', () => { - expect(parseCapability('weird')).toBeNull(); - }); -}); - -describe('supportsModel', () => { - it('thinking_level:xhigh maps to supports_xhigh', () => { - expect( - supportsModel(m({ supports_xhigh: true, supports_thinking: false }), { - type: 'thinking_level', - level: 'xhigh', - }), - ).toBe(true); - }); - - it('thinking_level:medium maps to supports_thinking', () => { - expect( - supportsModel(m({ supports_thinking: true, supports_xhigh: false }), { - type: 'thinking_level', - level: 'medium', - }), - ).toBe(true); - }); - - it('tools / vision / cache map to their flags', () => { - expect(supportsModel(m({ supports_tools: true }), { type: 'tools' })).toBe(true); - expect(supportsModel(m({ supports_vision: true }), { type: 'vision' })).toBe(true); - expect(supportsModel(m({ supports_cache: true }), { type: 'cache' })).toBe(true); - }); -}); diff --git a/harness/tests/provider-anthropic/auth.test.ts b/harness/tests/provider-anthropic/auth.test.ts deleted file mode 100644 index e155cf55c..000000000 --- a/harness/tests/provider-anthropic/auth.test.ts +++ /dev/null @@ -1,271 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { Model } from '../../src/models-catalog/types.js'; -import { - _resetProviderResolveCacheForTests, - buildConfig, - invalidateProviderResolveCache, -} from '../../src/provider-anthropic/auth.js'; -import type { WorkerConfig } from '../../src/provider-anthropic/config.js'; -import { streamAnthropic } from '../../src/provider-anthropic/stream.js'; -import { buildThinkingConfig } from '../../src/provider-anthropic/thinking.js'; -import type { AnthropicConfig } from '../../src/provider-anthropic/types.js'; -import type { ISdk } from '../../src/runtime/iii.js'; - -const WORKER: WorkerConfig = { - default_api_url: 'https://api.anthropic.com/v1/messages', - default_max_tokens: 8192, -}; - -const { resolveProviderMock } = vi.hoisted(() => ({ - resolveProviderMock: vi.fn(), -})); - -vi.mock('../../src/runtime/provider-resolve.js', () => ({ - resolveProvider: resolveProviderMock, -})); - -function resolved(max_tokens: number | null) { - return { - configured: true, - credential: { type: 'api_key', key: 'sk-test' }, - api_url: null, - max_tokens, - source: 'stored', - }; -} - -function iiiWithCatalog(entry: unknown, opts: { throws?: boolean } = {}): ISdk { - const trigger = vi.fn().mockImplementation(async (req: { function_id: string }) => { - if (req.function_id === 'models::get') { - if (opts.throws) throw new Error('bus timeout'); - return entry; - } - return null; - }); - return { trigger } as unknown as ISdk; -} - -function iiiCountingCatalog(entry: unknown): { iii: ISdk; modelsGetCalls: () => number } { - let n = 0; - const trigger = vi.fn().mockImplementation(async (req: { function_id: string }) => { - if (req.function_id === 'models::get') { - n++; - return entry; - } - return null; - }); - return { iii: { trigger } as unknown as ISdk, modelsGetCalls: () => n }; -} - -const THINKING_MODEL: Model = { - id: 'claude-sonnet-4-6', - provider: 'anthropic', - api: 'anthropic-messages', - display_name: 'Claude Sonnet 4.6', - context_window: 1_000_000, - max_output_tokens: 64_000, - supports_thinking: true, - supports_xhigh: true, - thinking_budgets: { minimal: 2_000, low: 4_000, medium: 8_000, high: 16_000 }, -}; - -describe('buildConfig max_tokens resolution', () => { - beforeEach(() => { - resolveProviderMock.mockReset(); - }); - - it('defaults to min(catalog max, 32k) — 64k model clamps to 32k', async () => { - resolveProviderMock.mockResolvedValue(resolved(null)); - const iii = iiiWithCatalog({ id: 'claude-sonnet-4-6', max_output_tokens: 64_000 }); - const cfg = await buildConfig(iii, WORKER, 'claude-sonnet-4-6'); - expect(cfg.max_tokens).toBe(32_000); - expect(cfg.catalog?.max_output_tokens).toBe(64_000); - }); - - it('uses the catalog max when below the cap', async () => { - resolveProviderMock.mockResolvedValue(resolved(null)); - const iii = iiiWithCatalog({ id: 'claude-haiku-4-5', max_output_tokens: 16_000 }); - const cfg = await buildConfig(iii, WORKER, 'claude-haiku-4-5'); - expect(cfg.max_tokens).toBe(16_000); - }); - - it('registry override wins below the model ceiling', async () => { - resolveProviderMock.mockResolvedValue(resolved(50_000)); - const iii = iiiWithCatalog({ id: 'claude-sonnet-4-6', max_output_tokens: 64_000 }); - const cfg = await buildConfig(iii, WORKER, 'claude-sonnet-4-6'); - expect(cfg.max_tokens).toBe(50_000); - }); - - it('registry override is clamped to the model ceiling', async () => { - resolveProviderMock.mockResolvedValue(resolved(100_000)); - const iii = iiiWithCatalog({ id: 'claude-sonnet-4-6', max_output_tokens: 64_000 }); - const cfg = await buildConfig(iii, WORKER, 'claude-sonnet-4-6'); - expect(cfg.max_tokens).toBe(64_000); - }); - - it('falls back to the registry/worker default when the catalog lookup fails', async () => { - resolveProviderMock.mockResolvedValue(resolved(null)); - const cfg = await buildConfig(iiiWithCatalog(null, { throws: true }), WORKER, 'claude-x'); - expect(cfg.max_tokens).toBe(WORKER.default_max_tokens); - expect(cfg.catalog).toBeUndefined(); - }); - - it('falls back to the worker default when the model is unknown', async () => { - resolveProviderMock.mockResolvedValue(resolved(null)); - const cfg = await buildConfig(iiiWithCatalog(null), WORKER, 'claude-unknown'); - expect(cfg.max_tokens).toBe(WORKER.default_max_tokens); - }); - - it('throws without a credential', async () => { - resolveProviderMock.mockResolvedValue({ - configured: false, - credential: null, - api_url: null, - max_tokens: null, - source: null, - }); - await expect(buildConfig(iiiWithCatalog(null), WORKER, 'claude-x')).rejects.toThrow( - /no credential/, - ); - }); -}); - -describe('buildConfig pre-resolved model threading', () => { - beforeEach(() => { - resolveProviderMock.mockReset(); - resolveProviderMock.mockResolvedValue(resolved(null)); - }); - - it('uses the pre-resolved model and skips models::get', async () => { - const { iii, modelsGetCalls } = iiiCountingCatalog({ id: 'nope', max_output_tokens: 1 }); - - const cfg = await buildConfig(iii, WORKER, 'claude-sonnet-4-6', THINKING_MODEL); - - expect(modelsGetCalls()).toBe(0); - expect(cfg.catalog).toEqual(THINKING_MODEL); - expect(cfg.max_tokens).toBe(32_000); - }); - - it('fetches models::get when no pre-resolved model is threaded', async () => { - const { iii, modelsGetCalls } = iiiCountingCatalog({ - id: 'claude-sonnet-4-6', - max_output_tokens: 64_000, - }); - - const cfg = await buildConfig(iii, WORKER, 'claude-sonnet-4-6'); - - expect(modelsGetCalls()).toBe(1); - expect(cfg.catalog?.id).toBe('claude-sonnet-4-6'); - }); - - it.each([ - 'high', - 'xhigh', - ])('produces a byte-identical thinking config (%s) with vs without the pre-resolved model', async (level) => { - const fetched = await buildConfig(iiiWithCatalog(THINKING_MODEL), WORKER, 'claude-sonnet-4-6'); - const { iii } = iiiCountingCatalog({ id: 'nope' }); - const threaded = await buildConfig(iii, WORKER, 'claude-sonnet-4-6', THINKING_MODEL); - - const fetchedThinking = buildThinkingConfig(level, fetched.max_tokens, fetched.catalog); - const threadedThinking = buildThinkingConfig(level, threaded.max_tokens, threaded.catalog); - - expect(threadedThinking).toEqual(fetchedThinking); - expect(threadedThinking).toBeDefined(); - }); -}); - -function anthropicCfg(): AnthropicConfig { - return { - credential_value: 'sk-test', - model: 'claude-sonnet-4-6', - max_tokens: 32_000, - api_url: 'https://api.example/v1/messages', - auth_mode: 'api_key', - }; -} - -describe('buildConfig per-turn credential resolution cache', () => { - beforeEach(() => { - resolveProviderMock.mockReset(); - resolveProviderMock.mockResolvedValue(resolved(null)); - _resetProviderResolveCacheForTests(); - }); - - it('resolves once per turn key and reuses it across streams', async () => { - const iii = iiiWithCatalog({ id: 'claude-sonnet-4-6', max_output_tokens: 64_000 }); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - expect(resolveProviderMock).toHaveBeenCalledTimes(1); - }); - - it('re-resolves when the turn key changes (next user turn)', async () => { - const iii = iiiWithCatalog({ id: 'claude-sonnet-4-6', max_output_tokens: 64_000 }); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 200); - expect(resolveProviderMock).toHaveBeenCalledTimes(2); - }); - - it('does not cache when no turn key is threaded', async () => { - const iii = iiiWithCatalog({ id: 'claude-sonnet-4-6', max_output_tokens: 64_000 }); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6'); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6'); - expect(resolveProviderMock).toHaveBeenCalledTimes(2); - }); - - it('invalidateProviderResolveCache forces a re-resolve on the same key', async () => { - const iii = iiiWithCatalog({ id: 'claude-sonnet-4-6', max_output_tokens: 64_000 }); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - invalidateProviderResolveCache(); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - expect(resolveProviderMock).toHaveBeenCalledTimes(2); - }); - - describe('401 invalidation via streamAnthropic', () => { - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it('a 401 stream drops the cache so the next turn re-resolves the credential', async () => { - const iii = iiiWithCatalog({ id: 'claude-sonnet-4-6', max_output_tokens: 64_000 }); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - expect(resolveProviderMock).toHaveBeenCalledTimes(1); - - vi.stubGlobal( - 'fetch', - vi.fn(async () => new Response('unauthorized', { status: 401 })), - ); - for await (const _ev of streamAnthropic({ - cfg: anthropicCfg(), - system_prompt: '', - messages: [], - tools: [], - })) { - } - - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - expect(resolveProviderMock).toHaveBeenCalledTimes(2); - }); - - it('a 200 stream leaves the cache intact', async () => { - const iii = iiiWithCatalog({ id: 'claude-sonnet-4-6', max_output_tokens: 64_000 }); - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - - vi.stubGlobal( - 'fetch', - vi.fn(async () => new Response('data: {"type":"message_stop"}\n\n', { status: 200 })), - ); - for await (const _ev of streamAnthropic({ - cfg: anthropicCfg(), - system_prompt: '', - messages: [], - tools: [], - })) { - } - - await buildConfig(iii, WORKER, 'claude-sonnet-4-6', undefined, 100); - expect(resolveProviderMock).toHaveBeenCalledTimes(1); - }); - }); -}); diff --git a/harness/tests/provider-anthropic/cache.test.ts b/harness/tests/provider-anthropic/cache.test.ts deleted file mode 100644 index ba5a3b56c..000000000 --- a/harness/tests/provider-anthropic/cache.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { applyMessagesCacheAnchor } from '../../src/provider-anthropic/cache.js'; - -type WireMsg = Record; - -function assistantTurn(content: Array>): WireMsg { - return { role: 'assistant', content }; -} - -describe('applyMessagesCacheAnchor', () => { - it('anchors cache_control on the last block of the last stable assistant turn', () => { - const wire: WireMsg[] = [ - { role: 'user', content: [{ type: 'text', text: 'hi' }] }, - assistantTurn([{ type: 'text', text: 'answer' }]), - ]; - applyMessagesCacheAnchor(wire); - const content = (wire[1] as { content: Array> }).content; - expect(content[0]?.cache_control).toEqual({ type: 'ephemeral' }); - }); - - it('skips trailing thinking blocks (cache_control is rejected on them)', () => { - const wire: WireMsg[] = [ - { role: 'user', content: [{ type: 'text', text: 'hi' }] }, - assistantTurn([ - { type: 'text', text: 'answer' }, - { type: 'thinking', thinking: 'why', signature: 'sig' }, - ]), - ]; - applyMessagesCacheAnchor(wire); - const content = (wire[1] as { content: Array> }).content; - expect(content[1]?.cache_control).toBeUndefined(); - expect(content[0]?.cache_control).toEqual({ type: 'ephemeral' }); - }); - - it('does nothing when the turn contains only thinking blocks', () => { - const wire: WireMsg[] = [ - { role: 'user', content: [{ type: 'text', text: 'hi' }] }, - assistantTurn([{ type: 'thinking', thinking: 'why', signature: 'sig' }]), - ]; - applyMessagesCacheAnchor(wire); - const content = (wire[1] as { content: Array> }).content; - expect(content[0]?.cache_control).toBeUndefined(); - }); -}); diff --git a/harness/tests/provider-anthropic/discover.test.ts b/harness/tests/provider-anthropic/discover.test.ts deleted file mode 100644 index e144e49ed..000000000 --- a/harness/tests/provider-anthropic/discover.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { discoverAndRegister } from '../../src/provider-anthropic/discover.js'; -import type { ISdk } from '../../src/runtime/iii.js'; -import type { WorkerConfig } from '../../src/provider-anthropic/config.js'; - -const WORKER: WorkerConfig = { - default_api_url: 'https://api.anthropic.com/v1/messages', - default_max_tokens: 8192, -}; - -const { resolveProviderMock } = vi.hoisted(() => ({ - resolveProviderMock: vi.fn(), -})); - -vi.mock('../../src/runtime/provider-resolve.js', () => ({ - resolveProvider: resolveProviderMock, -})); - -function makeIii() { - const trigger = vi.fn().mockResolvedValue({ ids: [], count: 0 }); - return { iii: { trigger } as unknown as ISdk, trigger }; -} - -function byFn(trigger: ReturnType, id: string) { - return (trigger.mock.calls as Array<[{ function_id: string; payload: unknown }]>).filter( - (c) => c[0].function_id === id, - ); -} - -describe('discoverAndRegister', () => { - beforeEach(() => { - resolveProviderMock.mockReset(); - }); - - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it('reconciles empty catalog when there is no credential', async () => { - resolveProviderMock.mockResolvedValue({ - configured: false, - credential: null, - api_url: null, - max_tokens: null, - source: null, - }); - const { iii, trigger } = makeIii(); - - const out = await discoverAndRegister(iii, WORKER); - expect(out).toEqual([]); - expect(byFn(trigger, 'models::reconcile')).toHaveLength(1); - expect(byFn(trigger, 'models::reconcile')[0][0].payload).toEqual({ - provider: 'anthropic', - models: [], - }); - }); - - it('reconciles empty catalog on upstream 401 (invalid api key)', async () => { - resolveProviderMock.mockResolvedValue({ - configured: true, - credential: { type: 'api_key', key: 'sk-bad' }, - api_url: WORKER.default_api_url, - max_tokens: 8192, - source: 'stored', - }); - globalThis.fetch = vi - .fn() - .mockResolvedValue( - new Response(JSON.stringify({ error: 'invalid' }), { status: 401 }), - ) as typeof globalThis.fetch; - - const { iii, trigger } = makeIii(); - const out = await discoverAndRegister(iii, WORKER); - expect(out).toEqual([]); - expect(byFn(trigger, 'models::reconcile')).toHaveLength(1); - expect(byFn(trigger, 'models::reconcile')[0][0].payload).toEqual({ - provider: 'anthropic', - models: [], - }); - }); - - it('does not reconcile on transient upstream 503', async () => { - resolveProviderMock.mockResolvedValue({ - configured: true, - credential: { type: 'api_key', key: 'sk-ok' }, - api_url: WORKER.default_api_url, - max_tokens: 8192, - source: 'stored', - }); - globalThis.fetch = vi - .fn() - .mockResolvedValue(new Response('unavailable', { status: 503 })) as typeof globalThis.fetch; - - const { iii, trigger } = makeIii(); - const out = await discoverAndRegister(iii, WORKER); - expect(out).toEqual([]); - expect(byFn(trigger, 'models::reconcile')).toHaveLength(0); - }); - - it('reconciles discovered models in one call on success', async () => { - resolveProviderMock.mockResolvedValue({ - configured: true, - credential: { type: 'api_key', key: 'sk-good' }, - api_url: WORKER.default_api_url, - max_tokens: 8192, - source: 'stored', - }); - globalThis.fetch = vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - data: [{ id: 'claude-sonnet-4-20250514', display_name: 'Claude Sonnet 4' }], - }), - { status: 200 }, - ), - ) as typeof globalThis.fetch; - - const { iii, trigger } = makeIii(); - const out = await discoverAndRegister(iii, WORKER); - expect(out).toEqual(['claude-sonnet-4-20250514']); - const reconcile = byFn(trigger, 'models::reconcile'); - expect(reconcile).toHaveLength(1); - const payload = reconcile[0][0].payload as { provider: string; models: unknown[] }; - expect(payload.provider).toBe('anthropic'); - expect(payload.models).toHaveLength(1); - expect((payload.models[0] as { id: string }).id).toBe('claude-sonnet-4-20250514'); - }); -}); diff --git a/harness/tests/provider-anthropic/sse.test.ts b/harness/tests/provider-anthropic/sse.test.ts deleted file mode 100644 index 80445199a..000000000 --- a/harness/tests/provider-anthropic/sse.test.ts +++ /dev/null @@ -1,236 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - buildPartial, - emptyPartial, - handleSseEvent, - mapStopReason, - mergeUsage, -} from '../../src/provider-anthropic/sse.js'; - -describe('mapStopReason', () => { - it('maps known values', () => { - expect(mapStopReason('end_turn')).toBe('end'); - expect(mapStopReason('max_tokens')).toBe('length'); - expect(mapStopReason('tool_use')).toBe('function_call'); - expect(mapStopReason('weird')).toBe('end'); - }); -}); - -describe('mergeUsage', () => { - it('accumulates input + output + cache fields', () => { - const u = { input: 0, output: 0, cache_read: 0, cache_write: 0 }; - mergeUsage({ input_tokens: 10, output_tokens: 20 }, u); - mergeUsage( - { - input_tokens: 5, - output_tokens: 6, - cache_read_input_tokens: 80, - cache_creation_input_tokens: 20, - }, - u, - ); - expect(u.input).toBe(15); - expect(u.output).toBe(26); - expect(u.cache_read).toBe(80); - expect(u.cache_write).toBe(20); - }); -}); - -describe('handleSseEvent', () => { - it('emits text_start + text_delta + text_end on a text content block', () => { - const state = emptyPartial(); - const out: string[] = []; - out.push( - ...handleSseEvent( - 'data: {"type":"content_block_start","content_block":{"type":"text"}}', - state, - 'm', - ).map((e) => e.type), - ); - out.push( - ...handleSseEvent( - 'data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"hello"}}', - state, - 'm', - ).map((e) => e.type), - ); - out.push( - ...handleSseEvent('data: {"type":"content_block_stop"}', state, 'm').map((e) => e.type), - ); - expect(out).toEqual(['text_start', 'text_delta', 'text_end']); - expect(state.text_blocks).toEqual(['hello']); - }); - - it('accumulates input_json_delta into the latest function call', () => { - const state = emptyPartial(); - handleSseEvent( - 'data: {"type":"content_block_start","content_block":{"type":"tool_use","id":"tc1","name":"shell__fs__ls"}}', - state, - 'm', - ); - handleSseEvent( - 'data: {"type":"content_block_delta","delta":{"type":"input_json_delta","partial_json":"{\\"path"}}', - state, - 'm', - ); - handleSseEvent( - 'data: {"type":"content_block_delta","delta":{"type":"input_json_delta","partial_json":"\\":\\"/tmp\\"}"}}', - state, - 'm', - ); - expect(state.function_calls[0]?.id).toBe('tc1'); - expect(state.function_calls[0]?.function_id).toBe('shell::fs::ls'); - expect(state.function_calls[0]?.args_json).toBe('{"path":"/tmp"}'); - }); - - it('records stop_reason from message_delta and emits stop event', () => { - const state = emptyPartial(); - handleSseEvent('data: {"type":"message_delta","delta":{"stop_reason":"tool_use"}}', state, 'm'); - expect(state.stop_reason).toBe('function_call'); - const events = handleSseEvent('data: {"type":"message_stop"}', state, 'm'); - expect(events[0]?.type).toBe('stop'); - }); - - it('returns no events on unparseable SSE', () => { - const state = emptyPartial(); - expect(handleSseEvent('event: bad', state, 'm')).toEqual([]); - expect(handleSseEvent('data: not-json', state, 'm')).toEqual([]); - }); - - it('buildPartial reflects accumulated text', () => { - const state = emptyPartial(); - state.text_blocks.push('hello'); - const partial = buildPartial(state, 'm'); - expect(partial.content[0]).toEqual({ type: 'text', text: 'hello' }); - }); -}); - -describe('handleSseEvent thinking blocks', () => { - it('emits thinking_start + thinking_delta + thinking_end in order', () => { - const state = emptyPartial(); - const out: string[] = []; - out.push( - ...handleSseEvent( - 'data: {"type":"content_block_start","content_block":{"type":"thinking"}}', - state, - 'm', - ).map((e) => e.type), - ); - out.push( - ...handleSseEvent( - 'data: {"type":"content_block_delta","delta":{"type":"thinking_delta","thinking":"let me think"}}', - state, - 'm', - ).map((e) => e.type), - ); - out.push( - ...handleSseEvent('data: {"type":"content_block_stop"}', state, 'm').map((e) => e.type), - ); - expect(out).toEqual(['thinking_start', 'thinking_delta', 'thinking_end']); - expect(state.thinking_blocks[0]?.text).toBe('let me think'); - }); - - it('accumulates signature_delta without emitting events', () => { - const state = emptyPartial(); - handleSseEvent( - 'data: {"type":"content_block_start","content_block":{"type":"thinking"}}', - state, - 'm', - ); - const events = handleSseEvent( - 'data: {"type":"content_block_delta","delta":{"type":"signature_delta","signature":"sig123"}}', - state, - 'm', - ); - expect(events).toEqual([]); - expect(state.thinking_blocks[0]?.signature).toBe('sig123'); - }); - - it('persists a leading ThinkingContent block in the assembled message', () => { - const state = emptyPartial(); - handleSseEvent( - 'data: {"type":"content_block_start","content_block":{"type":"thinking"}}', - state, - 'm', - ); - handleSseEvent( - 'data: {"type":"content_block_delta","delta":{"type":"thinking_delta","thinking":"why"}}', - state, - 'm', - ); - handleSseEvent('data: {"type":"content_block_stop"}', state, 'm'); - state.text_blocks.push('answer'); - const partial = buildPartial(state, 'm'); - expect(partial.content).toEqual([ - { type: 'thinking', text: 'why' }, - { type: 'text', text: 'answer' }, - ]); - }); - - it('emits the end event matching the open block kind (regression: tool_use end)', () => { - const state = emptyPartial(); - handleSseEvent( - 'data: {"type":"content_block_start","content_block":{"type":"tool_use","id":"t1","name":"x"}}', - state, - 'm', - ); - const stop = handleSseEvent('data: {"type":"content_block_stop"}', state, 'm'); - expect(stop[0]?.type).toBe('functioncall_end'); - }); - - it('text block after a thinking block still emits text_end', () => { - const state = emptyPartial(); - handleSseEvent( - 'data: {"type":"content_block_start","content_block":{"type":"thinking"}}', - state, - 'm', - ); - handleSseEvent('data: {"type":"content_block_stop"}', state, 'm'); - handleSseEvent( - 'data: {"type":"content_block_start","content_block":{"type":"text"}}', - state, - 'm', - ); - const stop = handleSseEvent('data: {"type":"content_block_stop"}', state, 'm'); - expect(stop[0]?.type).toBe('text_end'); - }); - - it('preserves interleaved thinking/tool_use arrival order in the assembled message', () => { - const state = emptyPartial(); - const feed = (line: string) => handleSseEvent(`data: ${line}`, state, 'm'); - // thinking₁ → tool_use₁ → thinking₂ → tool_use₂ (interleaved-thinking beta shape) - feed('{"type":"content_block_start","content_block":{"type":"thinking"}}'); - feed('{"type":"content_block_delta","delta":{"type":"thinking_delta","thinking":"plan A"}}'); - feed('{"type":"content_block_stop"}'); - feed('{"type":"content_block_start","content_block":{"type":"tool_use","id":"t1","name":"x"}}'); - feed('{"type":"content_block_stop"}'); - feed('{"type":"content_block_start","content_block":{"type":"thinking"}}'); - feed('{"type":"content_block_delta","delta":{"type":"thinking_delta","thinking":"plan B"}}'); - feed('{"type":"content_block_stop"}'); - feed('{"type":"content_block_start","content_block":{"type":"tool_use","id":"t2","name":"y"}}'); - feed('{"type":"content_block_stop"}'); - const partial = buildPartial(state, 'm'); - expect(partial.content.map((b) => b.type)).toEqual([ - 'thinking', - 'function_call', - 'thinking', - 'function_call', - ]); - expect(partial.content[0]).toMatchObject({ text: 'plan A' }); - expect(partial.content[2]).toMatchObject({ text: 'plan B' }); - }); - - it('redacted_thinking does not crash and emits thinking events', () => { - const state = emptyPartial(); - const start = handleSseEvent( - 'data: {"type":"content_block_start","content_block":{"type":"redacted_thinking","data":"opaque"}}', - state, - 'm', - ); - expect(start[0]?.type).toBe('thinking_start'); - const stop = handleSseEvent('data: {"type":"content_block_stop"}', state, 'm'); - expect(stop[0]?.type).toBe('thinking_end'); - // Opaque content is not persisted. - expect(buildPartial(state, 'm').content).toEqual([]); - }); -}); diff --git a/harness/tests/provider-anthropic/stream-request.test.ts b/harness/tests/provider-anthropic/stream-request.test.ts deleted file mode 100644 index 1683d2be0..000000000 --- a/harness/tests/provider-anthropic/stream-request.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { streamAnthropic } from '../../src/provider-anthropic/stream.js'; -import type { AnthropicConfig } from '../../src/provider-anthropic/types.js'; - -function cfg(overrides: Partial = {}): AnthropicConfig { - return { - credential_value: 'sk-test', - model: 'claude-sonnet-4-6', - max_tokens: 32_000, - api_url: 'https://api.example/v1/messages', - auth_mode: 'api_key', - ...overrides, - }; -} - -type Captured = { body: Record; headers: Record }; - -async function capture(args: Parameters[0]): Promise { - let captured: Captured | null = null; - vi.stubGlobal( - 'fetch', - vi.fn().mockImplementation(async (_url: string, init: RequestInit) => { - captured = { - body: JSON.parse(init.body as string) as Record, - headers: init.headers as Record, - }; - return new Response('data: {"type":"message_stop"}\n\n', { status: 200 }); - }), - ); - for await (const _ev of streamAnthropic(args)) { - // drain - } - if (!captured) throw new Error('fetch was not called'); - return captured; -} - -afterEach(() => { - vi.unstubAllGlobals(); -}); - -describe('streamAnthropic request body', () => { - it('sends the resolved max_tokens and no temperature', async () => { - const { body } = await capture({ cfg: cfg(), system_prompt: 's', messages: [], tools: [] }); - expect(body.max_tokens).toBe(32_000); - expect(body).not.toHaveProperty('temperature'); - expect(body).not.toHaveProperty('thinking'); - }); - - it('omits the anthropic-beta header when thinking is off', async () => { - const { headers } = await capture({ cfg: cfg(), system_prompt: 's', messages: [], tools: [] }); - expect(headers).not.toHaveProperty('anthropic-beta'); - }); - - it('sends thinking + interleaved-thinking beta header when enabled', async () => { - const { body, headers } = await capture({ - cfg: cfg(), - system_prompt: 's', - messages: [], - tools: [], - thinking: { type: 'enabled', budget_tokens: 16_000 }, - }); - expect(body.thinking).toEqual({ type: 'enabled', budget_tokens: 16_000 }); - expect(headers['anthropic-beta']).toBe('interleaved-thinking-2025-05-14'); - expect(body).not.toHaveProperty('temperature'); - }); - - it('keeps the thinking budget below max_tokens (invariant carried by the caller)', async () => { - const { body } = await capture({ - cfg: cfg({ max_tokens: 32_000 }), - system_prompt: 's', - messages: [], - tools: [], - thinking: { type: 'enabled', budget_tokens: 31_999 }, - }); - expect(body.max_tokens).toBe(32_000); - const thinking = body.thinking as { budget_tokens: number }; - expect(thinking.budget_tokens).toBeLessThan(body.max_tokens as number); - }); -}); diff --git a/harness/tests/provider-anthropic/thinking.test.ts b/harness/tests/provider-anthropic/thinking.test.ts deleted file mode 100644 index c4337bdcc..000000000 --- a/harness/tests/provider-anthropic/thinking.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { Model } from '../../src/models-catalog/types.js'; -import { buildThinkingConfig, MIN_THINKING_BUDGET } from '../../src/provider-anthropic/thinking.js'; - -function model(max_output_tokens?: number, extra: Partial = {}): Model { - return { - id: 'claude-sonnet-4-6', - provider: 'anthropic', - api: 'anthropic-messages', - display_name: 'Claude Sonnet 4.6', - context_window: 200_000, - ...(max_output_tokens !== undefined ? { max_output_tokens } : {}), - ...extra, - }; -} - -describe('buildThinkingConfig', () => { - it.each([undefined, 'off'])('returns undefined for level %s', (level) => { - expect(buildThinkingConfig(level, 32_000, model(64_000))).toBeUndefined(); - }); - - it('returns undefined when the model output ceiling is unknown', () => { - expect(buildThinkingConfig('high', 32_000, model())).toBeUndefined(); - expect(buildThinkingConfig('high', 32_000, undefined)).toBeUndefined(); - }); - - describe('formula budgets (no catalog budgets)', () => { - // high -> min(16_000, floor(output/2 - 1)) - it.each([ - [8_192, 4_095], - [32_000, 15_999], - [64_000, 16_000], - ])('high with output %i -> budget %i', (output, budget) => { - expect(buildThinkingConfig('high', 32_000, model(output))).toEqual({ - type: 'enabled', - budget_tokens: budget, - }); - }); - - // max/xhigh -> min(31_999, output - 1), then clamped to leave output room - it('xhigh with output 64_000 reserves output room below max_tokens', () => { - expect(buildThinkingConfig('xhigh', 32_000, model(64_000))).toEqual({ - type: 'enabled', - budget_tokens: 32_000 - 1_024, - }); - }); - - it('medium with output 64_000 -> min(8_000, output/4)', () => { - expect(buildThinkingConfig('medium', 32_000, model(64_000))).toEqual({ - type: 'enabled', - budget_tokens: 8_000, - }); - }); - - it('low with output 64_000 -> min(4_000, output/8)', () => { - expect(buildThinkingConfig('low', 32_000, model(64_000))).toEqual({ - type: 'enabled', - budget_tokens: 4_000, - }); - }); - }); - - it('prefers catalog thinking_budgets over the formula', () => { - const m = model(64_000, { thinking_budgets: { high: 20_000 } }); - expect(buildThinkingConfig('high', 32_000, m)).toEqual({ - type: 'enabled', - budget_tokens: 20_000, - }); - }); - - it('never returns budget >= max_tokens and always reserves output room', () => { - const cfg = buildThinkingConfig('xhigh', 16_000, model(64_000)); - expect(cfg).toEqual({ type: 'enabled', budget_tokens: 16_000 - 1_024 }); - }); - - it('drops thinking when the clamped budget falls below the API minimum', () => { - // max_tokens 2047 leaves budget 1023 (< MIN_THINKING_BUDGET) after the reserve - expect(buildThinkingConfig('high', 2_047, model(64_000))).toBeUndefined(); - expect(buildThinkingConfig('high', MIN_THINKING_BUDGET, model(64_000))).toBeUndefined(); - }); - - it('drops thinking for unknown levels', () => { - expect(buildThinkingConfig('turbo', 32_000, model(64_000))).toBeUndefined(); - }); - - it('drops thinking when the catalog says the model cannot think', () => { - expect( - buildThinkingConfig('high', 32_000, model(64_000, { supports_thinking: false })), - ).toBeUndefined(); - // unknown flag stays permissive - expect(buildThinkingConfig('high', 32_000, model(64_000))).toBeDefined(); - }); - - it('degrades xhigh to the high tier when the catalog says xhigh is unsupported', () => { - expect(buildThinkingConfig('xhigh', 32_000, model(64_000, { supports_xhigh: false }))).toEqual({ - type: 'enabled', - budget_tokens: 16_000, - }); - // unknown flag keeps the xhigh formula - expect(buildThinkingConfig('xhigh', 32_000, model(64_000))).toEqual({ - type: 'enabled', - budget_tokens: 32_000 - 1_024, - }); - }); -}); diff --git a/harness/tests/provider-anthropic/wire-messages.test.ts b/harness/tests/provider-anthropic/wire-messages.test.ts deleted file mode 100644 index 0d897a064..000000000 --- a/harness/tests/provider-anthropic/wire-messages.test.ts +++ /dev/null @@ -1,434 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - contentBlockToWire, - decodeToolName, - encodeToolName, - toWireMessages, -} from '../../src/provider-anthropic/wire-messages.js'; -import type { AgentMessage } from '../../src/types/agent-message.js'; -import type { ContentBlock } from '../../src/types/content.js'; - -describe('encode/decodeToolName', () => { - it('replaces :: with __', () => { - expect(encodeToolName('shell::fs::ls')).toBe('shell__fs__ls'); - expect(decodeToolName('shell__fs__ls')).toBe('shell::fs::ls'); - }); -}); - -describe('contentBlockToWire', () => { - it('encodes function_call as tool_use with encoded name', () => { - const out = contentBlockToWire({ - type: 'function_call', - id: 'tc1', - function_id: 'shell::exec', - arguments: { x: 1 }, - }); - expect(out).toEqual({ - type: 'tool_use', - id: 'tc1', - name: 'shell__exec', - input: { x: 1 }, - }); - }); - - it('skips other block kinds', () => { - expect( - contentBlockToWire({ - type: 'image', - mime: 'image/png', - data: 'aGk=', - }), - ).toBeNull(); - }); - - it('round-trips signed thinking blocks (required for thinking + tool use)', () => { - expect( - contentBlockToWire({ type: 'thinking', text: 'reasoning…', signature: 'sig123' }), - ).toEqual({ type: 'thinking', thinking: 'reasoning…', signature: 'sig123' }); - }); - - it('drops unsigned thinking blocks (would fail signature verification)', () => { - expect(contentBlockToWire({ type: 'thinking', text: 'partial' })).toBeNull(); - }); -}); - -describe('toWireMessages thinking round-trip', () => { - it('keeps the signed thinking block before tool_use in the assistant turn', () => { - const msgs: AgentMessage[] = [ - { role: 'user', content: [{ type: 'text', text: 'go' }], timestamp: 0 }, - { - role: 'assistant', - content: [ - { type: 'thinking', text: 'plan it', signature: 'sig' }, - { type: 'function_call', id: 't1', function_id: 'shell::exec', arguments: {} }, - ], - stop_reason: 'function_call', - error_message: null, - error_kind: null, - usage: null, - model: 'm', - provider: 'anthropic', - timestamp: 1, - }, - { - role: 'function_result', - function_call_id: 't1', - function_id: 'shell::exec', - content: [], - details: null, - is_error: false, - timestamp: 2, - }, - ]; - const wire = toWireMessages(msgs) as Array<{ role: string; content: Array<{ type: string }> }>; - const assistant = wire.find((m) => m.role === 'assistant'); - expect(assistant?.content.map((b) => b.type)).toEqual(['thinking', 'tool_use']); - }); -}); - -describe('toWireMessages', () => { - it('converts user message to wire user', () => { - const msgs: AgentMessage[] = [ - { role: 'user', content: [{ type: 'text', text: 'hi' }], timestamp: 0 }, - ]; - const wire = toWireMessages(msgs) as Array>; - expect(wire[0]?.role).toBe('user'); - expect((wire[0] as { content: Array<{ type: string }> }).content[0]?.type).toBe('text'); - }); - - it('collapses parallel function_results into one user message with tool_result blocks', () => { - const mk = (id: string): AgentMessage => ({ - role: 'function_result', - function_call_id: id, - function_id: 'read', - content: [{ type: 'text', text: id }], - details: {}, - is_error: false, - timestamp: 0, - }); - const wire = toWireMessages([mk('a'), mk('b'), mk('c')]) as Array>; - expect(wire.length).toBe(1); - const content = (wire[0] as { content: Array<{ tool_use_id: string }> }).content; - expect(content).toHaveLength(3); - expect(content[0]?.tool_use_id).toBe('a'); - expect(content[2]?.tool_use_id).toBe('c'); - }); - - it('skips custom messages', () => { - const msgs: AgentMessage[] = [ - { - role: 'custom', - custom_type: 'note', - content: [{ type: 'text', text: 'x' }], - timestamp: 0, - }, - ]; - expect(toWireMessages(msgs)).toEqual([]); - }); - - describe('tool_result image rendering', () => { - const mkResult = (content: ContentBlock[]): AgentMessage => ({ - role: 'function_result', - function_call_id: 'toolu_img', - function_id: 'web::fetch', - content, - details: {}, - is_error: false, - timestamp: 0, - }); - - it('keeps the flat-string content shape when the result has no images', () => { - const wire = toWireMessages([mkResult([{ type: 'text', text: 'plain' }])]) as Array< - Record - >; - const content = (wire[0] as { content: Array<{ content: unknown }> }).content; - expect(content[0]?.content).toBe('plain'); // string, NOT an array - }); - - it('switches to an array of text + image source blocks when an image is present', () => { - const wire = toWireMessages([ - mkResult([ - { type: 'image', mime: 'image/png', data: 'aGVsbG8=' }, - { type: 'text', text: 'Image fetched (image/png, 5 bytes)' }, - ]), - ]) as Array>; - const content = (wire[0] as { content: Array<{ content: unknown }> }).content; - expect(content[0]?.content).toEqual([ - { type: 'text', text: 'Image fetched (image/png, 5 bytes)' }, - { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'aGVsbG8=' } }, - ]); - }); - - it('renders an image-only result as a single image source block', () => { - const wire = toWireMessages([ - mkResult([{ type: 'image', mime: 'image/jpeg', data: 'eA==' }]), - ]) as Array>; - const content = (wire[0] as { content: Array<{ content: unknown }> }).content; - expect(content[0]?.content).toEqual([ - { type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: 'eA==' } }, - ]); - }); - }); - - describe('boundary dedup of duplicate tool_result blocks', () => { - // Production failure: messages.20.content.1: each tool_use must have a - // single result. Found multiple tool_result blocks with id: toolu_... - // Root cause is orchestrator re-entry. The wire layer also defends. - - const mkResult = (id: string, text: string): AgentMessage => ({ - role: 'function_result', - function_call_id: id, - function_id: 'shell::run', - content: [{ type: 'text', text }], - details: {}, - is_error: false, - timestamp: 0, - }); - - it('collapses two function_results with the same id into one tool_result block (latest wins)', () => { - const wire = toWireMessages([ - mkResult('toolu_01', 'first attempt'), - mkResult('toolu_01', 'second attempt'), - ]) as Array>; - expect(wire.length).toBe(1); - const content = (wire[0] as { content: Array<{ tool_use_id: string; content: string }> }) - .content; - expect(content).toHaveLength(1); - expect(content[0]?.tool_use_id).toBe('toolu_01'); - expect(content[0]?.content).toBe('second attempt'); - }); - - it('preserves order of distinct tool_use_ids while deduping repeats', () => { - const wire = toWireMessages([ - mkResult('a', 'A1'), - mkResult('b', 'B1'), - mkResult('a', 'A2'), - mkResult('c', 'C1'), - mkResult('b', 'B2'), - ]) as Array>; - const content = (wire[0] as { content: Array<{ tool_use_id: string; content: string }> }) - .content; - expect(content.map((b) => b.tool_use_id)).toEqual(['a', 'b', 'c']); - expect(content[0]?.content).toBe('A2'); - expect(content[1]?.content).toBe('B2'); - expect(content[2]?.content).toBe('C1'); - }); - - it('dedup is scoped to the pending batch (across-batch repeats stay separate)', () => { - // [function_result(a), assistant_msg, function_result(a)] produces - // two SEPARATE user messages each with tool_result(a) — correct - // wire format. Dedup must NOT collapse across the flush boundary. - const msgs: AgentMessage[] = [ - mkResult('a', 'r1'), - { - role: 'assistant', - content: [{ type: 'text', text: 'thinking…' }], - stop_reason: 'end', - model: 'claude', - provider: 'anthropic', - timestamp: 0, - }, - mkResult('a', 'r2'), - ]; - const wire = toWireMessages(msgs) as Array>; - expect(wire).toHaveLength(3); - expect((wire[0] as { role: string }).role).toBe('user'); - expect((wire[1] as { role: string }).role).toBe('assistant'); - expect((wire[2] as { role: string }).role).toBe('user'); - }); - }); - - describe('boundary sanitization of orphan tool_use blocks', () => { - // Production failure: messages.53: `tool_use` IDs were found without - // `tool_result` blocks immediately after: toolu_01D7RTIXU6PXH7ER9PSRHH98. - // Three orchestrator paths can leave a tool_use without its matching - // function_result in flat-state (abort during awaiting_approval, sync - // /compact mid-execution, prepared-call desync). The wire layer - // defends by injecting a synthetic tool_result placeholder. - - const mkAsstWithToolUse = (id: string, fnId = 'shell::run'): AgentMessage => ({ - role: 'assistant', - content: [ - { - type: 'function_call', - id, - function_id: fnId, - arguments: { command: 'echo hi' }, - }, - ], - stop_reason: 'function_call', - model: 'claude', - provider: 'anthropic', - timestamp: 0, - }); - - const mkResult = (id: string, text: string): AgentMessage => ({ - role: 'function_result', - function_call_id: id, - function_id: 'shell::run', - content: [{ type: 'text', text }], - details: {}, - is_error: false, - timestamp: 0, - }); - - it('injects a synthetic tool_result placeholder for an orphan tool_use at end of conversation', async () => { - // Exact production shape: flat-state ends with assistant(tool_use) - // because handleFinalize never ran for that call. - const msgs: AgentMessage[] = [ - { - role: 'user', - content: [{ type: 'text', text: 'do a thing' }], - timestamp: 0, - }, - mkAsstWithToolUse('toolu_orphan'), - ]; - const wire = toWireMessages(msgs) as Array>; - // [user, assistant(tool_use), user(synthetic tool_result)] - expect(wire).toHaveLength(3); - expect((wire[2] as { role: string }).role).toBe('user'); - const content = (wire[2] as { content: Array> }).content; - expect(content).toHaveLength(1); - expect(content[0]?.type).toBe('tool_result'); - expect(content[0]?.tool_use_id).toBe('toolu_orphan'); - expect(content[0]?.is_error).toBe(true); - expect(content[0]?.content as string).toMatch(/interrupted before completing/i); - }); - - it('does NOT inject anything when every tool_use already has a function_result (no-op happy path)', () => { - const msgs: AgentMessage[] = [ - { - role: 'user', - content: [{ type: 'text', text: 'hi' }], - timestamp: 0, - }, - mkAsstWithToolUse('toolu_a'), - mkResult('toolu_a', 'output A'), - ]; - const wire = toWireMessages(msgs) as Array>; - expect(wire).toHaveLength(3); - const content = (wire[2] as { content: Array> }).content; - expect(content).toHaveLength(1); - expect(content[0]?.tool_use_id).toBe('toolu_a'); - expect(content[0]?.is_error).toBe(false); - // The synthetic placeholder text must NOT appear. - expect(content[0]?.content as string).not.toMatch(/interrupted before completing/i); - }); - - it('mixes synthetic placeholders alongside real tool_results when only SOME tool_uses are orphaned', () => { - // Assistant emits two tool_uses in one turn; only one runs to - // completion, the other was interrupted. - const msgs: AgentMessage[] = [ - { - role: 'user', - content: [{ type: 'text', text: 'do both' }], - timestamp: 0, - }, - { - role: 'assistant', - content: [ - { - type: 'function_call', - id: 'toolu_done', - function_id: 'shell::run', - arguments: {}, - }, - { - type: 'function_call', - id: 'toolu_orphan', - function_id: 'shell::run', - arguments: {}, - }, - ], - stop_reason: 'function_call', - model: 'claude', - provider: 'anthropic', - timestamp: 0, - }, - mkResult('toolu_done', 'completed output'), - ]; - const wire = toWireMessages(msgs) as Array>; - expect(wire).toHaveLength(3); - const content = (wire[2] as { content: Array> }).content; - expect(content).toHaveLength(2); - const byId = new Map(content.map((b) => [b.tool_use_id as string, b])); - expect(byId.get('toolu_done')?.is_error).toBe(false); - expect(byId.get('toolu_done')?.content).toBe('completed output'); - expect(byId.get('toolu_orphan')?.is_error).toBe(true); - expect(byId.get('toolu_orphan')?.content as string).toMatch(/interrupted before completing/i); - }); - - it('injects placeholders in turn-order across multiple orphan turns', () => { - // Long agentic session where two earlier turns each left an orphan. - // The synthetic for toolu_x merges into the next user message - // (with q2's text), preventing consecutive user messages. - const msgs: AgentMessage[] = [ - { - role: 'user', - content: [{ type: 'text', text: 'q1' }], - timestamp: 0, - }, - mkAsstWithToolUse('toolu_x'), - { - role: 'user', - content: [{ type: 'text', text: 'q2' }], - timestamp: 0, - }, - mkAsstWithToolUse('toolu_y'), - ]; - const wire = toWireMessages(msgs) as Array>; - expect(wire.map((m) => (m as { role: string }).role)).toEqual([ - 'user', // q1 - 'assistant', // tool_use_x - 'user', // synthetic tool_result_x + q2 text MERGED - 'assistant', // tool_use_y - 'user', // synthetic tool_result_y at end - ]); - // The merged user at index 2 carries BOTH the synthetic placeholder - // AND the q2 text content. - const m2 = wire[2] as { content: Array> }; - const placeholderBlock = m2.content.find( - (b) => - b.type === 'tool_result' && (b as { tool_use_id?: string }).tool_use_id === 'toolu_x', - ); - expect(placeholderBlock).toBeDefined(); - const q2TextBlock = m2.content.find( - (b) => b.type === 'text' && (b as { text?: string }).text === 'q2', - ); - expect(q2TextBlock).toBeDefined(); - // Tool_result must come BEFORE the user text (Anthropic convention). - const placeholderIdx = m2.content.findIndex((b) => b.type === 'tool_result'); - const textIdx = m2.content.findIndex((b) => b.type === 'text'); - expect(placeholderIdx).toBeLessThan(textIdx); - // The final user (after toolu_y assistant) carries the toolu_y placeholder. - const m4 = wire[4] as { content: Array> }; - expect((m4.content[0] as { tool_use_id: string }).tool_use_id).toBe('toolu_y'); - }); - - it('does not re-inject a placeholder if the same orphan id appears twice (defensive)', () => { - // Pathological: same tool_use_id in two assistant turns and no - // function_result for either. The first injection marks the id - // resolved so the second skips. Anthropic gets one placeholder for - // the id; the second tool_use will be unmatched in its own batch - // but at least we don't generate two placeholders. - const msgs: AgentMessage[] = [mkAsstWithToolUse('toolu_dup'), mkAsstWithToolUse('toolu_dup')]; - const wire = toWireMessages(msgs) as Array>; - let placeholderCount = 0; - for (const m of wire) { - const role = (m as { role: string }).role; - if (role !== 'user') continue; - const content = (m as { content: Array> }).content; - for (const b of content) { - if ( - b.type === 'tool_result' && - typeof b.content === 'string' && - /interrupted/i.test(b.content) - ) { - placeholderCount++; - } - } - } - expect(placeholderCount).toBe(1); - }); - }); -}); diff --git a/harness/tests/provider-kimi/sse.test.ts b/harness/tests/provider-kimi/sse.test.ts deleted file mode 100644 index a6200f1b5..000000000 --- a/harness/tests/provider-kimi/sse.test.ts +++ /dev/null @@ -1,216 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - buildPartial, - classifyKimiError, - emptyPartial, - handleChunk, - mapFinishReason, - mergeUsage, -} from '../../src/provider-kimi/sse.js'; - -describe('mergeUsage (kimi)', () => { - it('extracts chat-completions cached_tokens', () => { - const u = { input: 0, output: 0, cache_read: 0, cache_write: 0 }; - mergeUsage( - { - prompt_tokens: 1500, - completion_tokens: 200, - prompt_tokens_details: { cached_tokens: 1200 }, - }, - u, - ); - expect(u.input).toBe(1500); - expect(u.output).toBe(200); - expect(u.cache_read).toBe(1200); - }); -}); - -describe('mapFinishReason (kimi)', () => { - it('maps known finish reasons', () => { - expect(mapFinishReason('stop')).toBe('end'); - expect(mapFinishReason('length')).toBe('length'); - expect(mapFinishReason('tool_calls')).toBe('function_call'); - expect(mapFinishReason('function_call')).toBe('function_call'); - }); -}); - -describe('handleChunk (kimi)', () => { - it('emits text_start on first content delta then text_delta', () => { - const state = emptyPartial(); - const events = handleChunk( - { choices: [{ delta: { content: 'hi' } }] }, - state, - 'kimi-k2-0905-preview', - 'kimi', - ); - expect(events.map((e) => e.type)).toEqual(['text_start', 'text_delta']); - expect(state.text).toBe('hi'); - }); - - it('accumulates tool_call arguments across chunks', () => { - const state = emptyPartial(); - handleChunk( - { - choices: [ - { - delta: { - tool_calls: [ - { index: 0, id: 'tc1', function: { name: 'shell::exec', arguments: '{"x' } }, - ], - }, - }, - ], - }, - state, - 'kimi-k2-0905-preview', - 'kimi', - ); - handleChunk( - { - choices: [{ delta: { tool_calls: [{ index: 0, function: { arguments: '":1}' } }] } }], - }, - state, - 'kimi-k2-0905-preview', - 'kimi', - ); - expect(state.tool_calls[0]?.id).toBe('tc1'); - expect(state.tool_calls[0]?.function_id).toBe('shell::exec'); - expect(state.tool_calls[0]?.args_json).toBe('{"x":1}'); - }); - - it('records finish_reason as stop_reason on the partial state', () => { - const state = emptyPartial(); - handleChunk( - { choices: [{ finish_reason: 'tool_calls' }] }, - state, - 'kimi-k2-0905-preview', - 'kimi', - ); - const partial = buildPartial(state, 'kimi-k2-0905-preview', 'kimi'); - expect(partial.stop_reason).toBe('function_call'); - }); -}); - -describe('classifyKimiError', () => { - it('maps 401/403 to auth_expired', () => { - expect(classifyKimiError('unauthorized', 401)).toBe('auth_expired'); - expect(classifyKimiError('forbidden', 403)).toBe('auth_expired'); - }); - - it('maps 429 to rate_limited', () => { - expect(classifyKimiError('too many requests', 429)).toBe('rate_limited'); - }); - - it('maps 5xx to transient', () => { - expect(classifyKimiError('bad gateway', 502)).toBe('transient'); - }); - - it('maps context length messages to context_overflow', () => { - expect(classifyKimiError('context length exceeded', 400)).toBe('context_overflow'); - }); - - it('defaults to permanent', () => { - expect(classifyKimiError('something else', 400)).toBe('permanent'); - }); -}); - -describe('handleChunk — Kimi K2 thinking mode reasoning_content', () => { - // Kimi K2.6 streams reasoning tokens on `delta.reasoning_content` BEFORE - // any content/tool_calls. If we don't capture them and don't echo back - // on the next request, Kimi rejects: "thinking is enabled but - // reasoning_content is missing in assistant tool call message". - - it('accumulates delta.reasoning_content into state.reasoning_text and emits thinking_start + thinking_delta', () => { - const state = emptyPartial(); - const events = handleChunk( - { choices: [{ delta: { reasoning_content: 'Let me think about this' } }] }, - state, - 'kimi-k2.6', - 'kimi', - ); - expect(state.reasoning_text).toBe('Let me think about this'); - expect(events.map((e) => e.type)).toEqual(['thinking_start', 'thinking_delta']); - const delta = events[1] as Extract<(typeof events)[number], { type: 'thinking_delta' }>; - expect(delta.delta).toBe('Let me think about this'); - }); - - it('emits thinking_delta only (no second thinking_start) on subsequent reasoning chunks', () => { - const state = emptyPartial(); - handleChunk({ choices: [{ delta: { reasoning_content: 'first ' } }] }, state, 'm', 'kimi'); - const events = handleChunk( - { choices: [{ delta: { reasoning_content: 'second' } }] }, - state, - 'm', - 'kimi', - ); - expect(state.reasoning_text).toBe('first second'); - expect(events.map((e) => e.type)).toEqual(['thinking_delta']); - }); - - it('persists reasoning_text as a leading thinking ContentBlock on the AssistantMessage', () => { - const state = emptyPartial(); - handleChunk( - { choices: [{ delta: { reasoning_content: 'plan: call shell::ls' } }] }, - state, - 'kimi-k2.6', - 'kimi', - ); - handleChunk({ choices: [{ delta: { content: 'ok' } }] }, state, 'kimi-k2.6', 'kimi'); - const partial = buildPartial(state, 'kimi-k2.6', 'kimi'); - // Thinking must come FIRST so wire-messages can project it back as - // reasoning_content on the assistant entry before content/tool_calls. - expect(partial.content[0]).toEqual({ type: 'thinking', text: 'plan: call shell::ls' }); - expect(partial.content[1]).toEqual({ type: 'text', text: 'ok' }); - }); - - it('handles reasoning_content + tool_calls in the same turn (the failing-on-prod shape)', () => { - const state = emptyPartial(); - handleChunk( - { - choices: [ - { - delta: { - reasoning_content: 'I need to check the engine functions', - tool_calls: [ - { - index: 0, - id: 'tc-1', - function: { name: 'directory::engine::functions::list', arguments: '{}' }, - }, - ], - }, - }, - ], - }, - state, - 'kimi-k2.6', - 'kimi', - ); - const partial = buildPartial(state, 'kimi-k2.6', 'kimi'); - // Critical assertion for the bug: tool-call turn must carry the - // thinking block, otherwise wire-messages can't echo it back as - // reasoning_content and Kimi rejects the next request. - const thinking = partial.content.find((c) => c.type === 'thinking') as - | { type: 'thinking'; text: string } - | undefined; - expect(thinking?.text).toBe('I need to check the engine functions'); - const fcall = partial.content.find((c) => c.type === 'function_call') as - | { type: 'function_call'; function_id: string } - | undefined; - expect(fcall?.function_id).toBe('directory::engine::functions::list'); - }); - - it('emits nothing for thinking when reasoning_content is empty string', () => { - const state = emptyPartial(); - const events = handleChunk( - { choices: [{ delta: { reasoning_content: '', content: 'real text' } }] }, - state, - 'm', - 'kimi', - ); - expect(state.reasoning_text).toBe(''); - // text_start + text_delta from the real content; no thinking_* events. - expect(events.some((e) => e.type === 'thinking_start')).toBe(false); - expect(events.some((e) => e.type === 'thinking_delta')).toBe(false); - }); -}); diff --git a/harness/tests/provider-kimi/stream.test.ts b/harness/tests/provider-kimi/stream.test.ts deleted file mode 100644 index 86381618e..000000000 --- a/harness/tests/provider-kimi/stream.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { collect, streamKimi } from '../../src/provider-kimi/stream.js'; -import type { ChatCompletionsConfig } from '../../src/provider-kimi/types.js'; - -const cfg: ChatCompletionsConfig = { - url: 'https://api.moonshot.ai/v1/chat/completions', - provider_name: 'kimi', - model: 'kimi-k2-0905-preview', - api_key: 'sk-test', - max_tokens: 256, -}; - -function sseResponse(chunks: string[], status = 200): Response { - const encoder = new TextEncoder(); - const stream = new ReadableStream({ - start(controller) { - for (const c of chunks) controller.enqueue(encoder.encode(c)); - controller.close(); - }, - }); - return new Response(stream, { - status, - headers: { 'content-type': 'text/event-stream' }, - }); -} - -function errorResponse(status: number, body: string): Response { - return new Response(body, { status }); -} - -describe('streamKimi', () => { - let originalFetch: typeof globalThis.fetch; - - beforeEach(() => { - originalFetch = globalThis.fetch; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - }); - - it('emits start -> text_start -> text_delta+ -> done on a happy-path stream', async () => { - globalThis.fetch = vi - .fn() - .mockResolvedValue( - sseResponse([ - 'data: {"choices":[{"delta":{"content":"Hel"}}]}\n\n', - 'data: {"choices":[{"delta":{"content":"lo"}}]}\n\n', - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]), - ); - - const events: string[] = []; - let finalText = ''; - for await (const ev of streamKimi({ cfg, system_prompt: '', messages: [], tools: [] })) { - events.push(ev.type); - if (ev.type === 'done') { - finalText = ev.message.content - .filter((c): c is { type: 'text'; text: string } => c.type === 'text') - .map((c) => c.text) - .join(''); - } - } - expect(events).toEqual(['start', 'text_start', 'text_delta', 'text_delta', 'done']); - expect(finalText).toBe('Hello'); - }); - - it('classifies HTTP 401 as auth_expired', async () => { - globalThis.fetch = vi.fn().mockResolvedValue(errorResponse(401, 'unauthorized')); - const final = await collect(streamKimi({ cfg, system_prompt: '', messages: [], tools: [] })); - expect(final.stop_reason).toBe('error'); - expect(final.error_kind).toBe('auth_expired'); - }); - - it('classifies HTTP 429 as rate_limited', async () => { - globalThis.fetch = vi.fn().mockResolvedValue(errorResponse(429, 'slow down')); - const final = await collect(streamKimi({ cfg, system_prompt: '', messages: [], tools: [] })); - expect(final.error_kind).toBe('rate_limited'); - }); - - it('classifies HTTP 500 as transient', async () => { - globalThis.fetch = vi.fn().mockResolvedValue(errorResponse(503, 'service unavailable')); - const final = await collect(streamKimi({ cfg, system_prompt: '', messages: [], tools: [] })); - expect(final.error_kind).toBe('transient'); - }); - - it('surfaces fetch transport failures as a single error event', async () => { - globalThis.fetch = vi.fn().mockRejectedValue(new Error('ECONNREFUSED')); - const final = await collect(streamKimi({ cfg, system_prompt: '', messages: [], tools: [] })); - expect(final.stop_reason).toBe('error'); - expect(final.error_message).toContain('kimi fetch failed'); - }); - - it('sends Bearer token and JSON body to the configured URL', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue( - sseResponse([ - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]), - ); - globalThis.fetch = fetchMock; - - await collect(streamKimi({ cfg, system_prompt: 'sys', messages: [], tools: [] })); - - expect(fetchMock).toHaveBeenCalledTimes(1); - const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; - expect(url).toBe(cfg.url); - const headers = init.headers as Record; - expect(headers.Authorization).toBe('Bearer sk-test'); - expect(headers['content-type']).toBe('application/json'); - const body = JSON.parse(init.body as string) as Record; - expect(body.model).toBe(cfg.model); - expect(body.stream).toBe(true); - expect((body.messages as Array<{ role: string }>)[0]?.role).toBe('system'); - }); -}); diff --git a/harness/tests/provider-kimi/wire-messages.test.ts b/harness/tests/provider-kimi/wire-messages.test.ts deleted file mode 100644 index 6f6f00447..000000000 --- a/harness/tests/provider-kimi/wire-messages.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { toOpenaiMessages } from '../../src/provider-kimi/wire-messages.js'; -import type { AgentMessage } from '../../src/types/agent-message.js'; - -describe('toOpenaiMessages (kimi)', () => { - it('prepends system message when present', () => { - const out = toOpenaiMessages([], 'be helpful') as Array>; - expect(out[0]).toEqual({ role: 'system', content: 'be helpful' }); - }); - - it('encodes assistant tool_calls with stringified arguments', () => { - const msg: AgentMessage = { - role: 'assistant', - content: [ - { type: 'text', text: 'calling' }, - { - type: 'function_call', - id: 'tc1', - function_id: 'shell::fs::ls', - arguments: { path: '/tmp' }, - }, - ], - stop_reason: 'function_call', - model: 'kimi-k2-0905-preview', - provider: 'kimi', - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - expect(out[0]?.role).toBe('assistant'); - const tcs = (out[0] as { tool_calls: Array<{ function: { arguments: string } }> }).tool_calls; - expect(tcs[0]?.function.arguments).toBe('{"path":"/tmp"}'); - }); - - it('emits tool messages with content + tool_call_id + is_error', () => { - const out = toOpenaiMessages( - [ - { - role: 'function_result', - function_call_id: 'tc1', - function_id: 'read', - content: [{ type: 'text', text: 'ok' }], - details: { status: 'denied' }, - is_error: true, - timestamp: 0, - }, - ], - '', - ) as Array>; - expect(out[0]?.role).toBe('tool'); - expect(out[0]?.tool_call_id).toBe('tc1'); - expect(out[0]?.is_error).toBe(true); - expect((out[0]?.content as string).startsWith('[PERMISSION_DENIED]')).toBe(true); - }); - - it('joins user text content with newlines', () => { - const msg: AgentMessage = { - role: 'user', - content: [ - { type: 'text', text: 'one' }, - { type: 'text', text: 'two' }, - ], - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - expect(out[0]?.content).toBe('one\ntwo'); - }); -}); - -describe('toOpenaiMessages (kimi) — round-tripping reasoning_content', () => { - // The bug this fixes: Kimi K2.6 thinking mode requires `reasoning_content` - // on every assistant message that has tool_calls. Without it Kimi rejects: - // "thinking is enabled but reasoning_content is missing in assistant - // tool call message at index N" - // sse.ts captures `delta.reasoning_content` into a ThinkingContent block; - // wire-messages.ts (this code) must project it back as `reasoning_content`. - - it('emits reasoning_content on assistant messages whose content has a thinking block + tool_calls', () => { - const msg: AgentMessage = { - role: 'assistant', - content: [ - { type: 'thinking', text: 'I need to list the engine functions first.' }, - { - type: 'function_call', - id: 'tc-1', - function_id: 'directory::engine::functions::list', - arguments: {}, - }, - ], - stop_reason: 'function_call', - model: 'kimi-k2.6', - provider: 'kimi', - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - const entry = out[0] as Record; - expect(entry.role).toBe('assistant'); - expect(entry.reasoning_content).toBe('I need to list the engine functions first.'); - expect(entry.tool_calls).toBeDefined(); - }); - - it('preserves reasoning_content + text + tool_calls together (the full assistant turn shape)', () => { - const msg: AgentMessage = { - role: 'assistant', - content: [ - { type: 'thinking', text: 'plan: A then B' }, - { type: 'text', text: 'On it.' }, - { - type: 'function_call', - id: 'tc-1', - function_id: 'shell::ls', - arguments: { path: '/' }, - }, - ], - stop_reason: 'function_call', - model: 'kimi-k2.6', - provider: 'kimi', - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - const entry = out[0] as Record; - expect(entry.reasoning_content).toBe('plan: A then B'); - expect(entry.content).toBe('On it.'); - expect(entry.tool_calls).toBeDefined(); - }); - - it('omits reasoning_content when no thinking block is present (non-thinking models)', () => { - const msg: AgentMessage = { - role: 'assistant', - content: [{ type: 'text', text: 'plain reply' }], - stop_reason: 'end', - model: 'kimi-k2-0905-preview', - provider: 'kimi', - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - const entry = out[0] as Record; - expect(entry.reasoning_content).toBeUndefined(); - }); - - it('concatenates multiple thinking blocks (rare, but defensive)', () => { - const msg: AgentMessage = { - role: 'assistant', - content: [ - { type: 'thinking', text: 'first thought. ' }, - { type: 'thinking', text: 'second thought.' }, - { type: 'text', text: 'reply' }, - ], - stop_reason: 'end', - model: 'kimi-k2.6', - provider: 'kimi', - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - expect((out[0] as Record).reasoning_content).toBe( - 'first thought. second thought.', - ); - }); - - it('end-to-end roundtrip: 3-message conversation with assistant tool_calls carries reasoning_content', () => { - // This is the exact shape that was failing in prod: - // index 0: system - // index 1: user - // index 2: assistant{tool_calls} — MUST carry reasoning_content - // index 3: tool result - const messages: AgentMessage[] = [ - { role: 'user', content: [{ type: 'text', text: 'create a sandbox' }], timestamp: 0 }, - { - role: 'assistant', - content: [ - { type: 'thinking', text: 'first I check what exists' }, - { - type: 'function_call', - id: 'tc-a', - function_id: 'directory::skills::get', - arguments: {}, - }, - ], - stop_reason: 'function_call', - model: 'kimi-k2.6', - provider: 'kimi', - timestamp: 0, - }, - { - role: 'function_result', - function_call_id: 'tc-a', - function_id: 'directory::skills::get', - content: [{ type: 'text', text: '[]' }], - details: {}, - is_error: false, - timestamp: 0, - }, - ]; - const out = toOpenaiMessages(messages, 'you are an agent') as Array>; - // [system, user, assistant, tool] - expect(out).toHaveLength(4); - expect(out[0]?.role).toBe('system'); - expect(out[1]?.role).toBe('user'); - expect(out[2]?.role).toBe('assistant'); - expect(out[2]?.reasoning_content).toBe('first I check what exists'); - expect(out[2]?.tool_calls).toBeDefined(); - expect(out[3]?.role).toBe('tool'); - }); -}); - -describe('toOpenaiMessages (kimi) — boundary dedup of duplicate tool messages', () => { - // Same defense as the Anthropic and OpenAI providers — never ship two - // tool messages with the same tool_call_id. Kimi's strict templates in - // K2.6 thinking mode reject this kind of duplicate. - - const mkResult = (id: string, text: string): AgentMessage => ({ - role: 'function_result', - function_call_id: id, - function_id: 'shell::run', - content: [{ type: 'text', text }], - details: {}, - is_error: false, - timestamp: 0, - }); - - it('keeps exactly one tool message per tool_call_id (latest wins)', () => { - const out = toOpenaiMessages( - [mkResult('call_01', 'first'), mkResult('call_01', 'second')], - '', - ) as Array>; - const tools = out.filter((m) => m.role === 'tool'); - expect(tools).toHaveLength(1); - expect(tools[0]?.content).toBe('second'); - }); -}); diff --git a/harness/tests/provider-llamacpp/auth.test.ts b/harness/tests/provider-llamacpp/auth.test.ts deleted file mode 100644 index db59cbe2b..000000000 --- a/harness/tests/provider-llamacpp/auth.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { - buildAuthHeaders, - buildConfig, - isLoopbackUrl, - selectAuthKey, -} from '../../src/provider-llamacpp/auth.js'; -import type { WorkerConfig } from '../../src/provider-llamacpp/config.js'; -import type { ISdk } from '../../src/runtime/iii.js'; -import type { Credential } from '../../src/runtime/provider-resolve.js'; - -const worker: WorkerConfig = { - default_max_tokens: 8192, - default_api_url: 'http://localhost:8080/v1/chat/completions', -}; - -/** Wrap a credential into the `harness::provider::resolve` result shape. */ -function resolveResult(cred: Credential | null) { - return { - configured: cred !== null, - source: cred ? 'stored' : null, - credential: cred, - api_url: null, - max_tokens: null, - }; -} - -function makeSdk(triggerImpl: (req: { function_id: string; payload: unknown }) => unknown): ISdk { - return { - trigger: vi.fn(triggerImpl), - registerFunction: vi.fn(), - } as unknown as ISdk; -} - -describe('buildConfig (llamacpp)', () => { - it('returns an empty api_key when no credential is stored (loopback)', async () => { - const sdk = makeSdk(() => resolveResult(null)); - const cfg = await buildConfig(sdk, worker, 'Meta-Llama-3-8B'); - // Unlike LM Studio there is no fallback bearer; the empty string - // is propagated and stream.ts omits the Authorization header. - expect(cfg.api_key).toBe(''); - expect(cfg.url).toBe(worker.default_api_url); - expect(cfg.provider_name).toBe('llamacpp'); - expect(cfg.model).toBe('Meta-Llama-3-8B'); - expect(cfg.max_tokens).toBe(worker.default_max_tokens); - }); - - it('returns an empty api_key when resolve throws', async () => { - const sdk = makeSdk(() => { - throw new Error('harness unreachable'); - }); - const cfg = await buildConfig(sdk, worker, 'Meta-Llama-3-8B'); - expect(cfg.api_key).toBe(''); - }); - - it('honours a real API key when one is configured on an authenticated llama-server', async () => { - const sdk = makeSdk(() => resolveResult({ type: 'api_key', key: 'sk-real-key' })); - const cfg = await buildConfig(sdk, worker, 'Meta-Llama-3-8B'); - expect(cfg.api_key).toBe('sk-real-key'); - }); - - it('resolves via harness::provider::resolve with provider="llamacpp"', async () => { - const trigger = vi.fn().mockResolvedValue(resolveResult(null)); - const sdk = { trigger, registerFunction: vi.fn() } as unknown as ISdk; - await buildConfig(sdk, worker, 'Meta-Llama-3-8B'); - expect(trigger).toHaveBeenCalledWith( - expect.objectContaining({ - function_id: 'harness::provider::resolve', - payload: { provider: 'llamacpp' }, - }), - ); - }); -}); - -describe('isLoopbackUrl', () => { - it('accepts canonical loopback hosts', () => { - expect(isLoopbackUrl('http://localhost:8080/v1/chat/completions')).toBe(true); - expect(isLoopbackUrl('http://127.0.0.1:8080/v1/chat/completions')).toBe(true); - expect(isLoopbackUrl('http://127.0.0.5/v1/chat/completions')).toBe(true); - expect(isLoopbackUrl('http://[::1]/v1/chat/completions')).toBe(true); - }); - - it('rejects non-loopback hosts', () => { - expect(isLoopbackUrl('http://example.com/v1/chat/completions')).toBe(false); - expect(isLoopbackUrl('http://192.168.1.10/v1/chat/completions')).toBe(false); - expect(isLoopbackUrl('https://my-tunnel.ngrok.io/v1/chat/completions')).toBe(false); - }); - - it('rejects malformed URLs (fail closed)', () => { - expect(isLoopbackUrl('not-a-url')).toBe(false); - expect(isLoopbackUrl('')).toBe(false); - }); -}); - -describe('selectAuthKey', () => { - it('returns the explicit key when one is configured', () => { - expect(selectAuthKey({ type: 'api_key', key: 'sk-explicit' }, 'http://localhost:8080/')).toBe( - 'sk-explicit', - ); - expect(selectAuthKey({ type: 'api_key', key: 'sk-explicit' }, 'https://remote.example/')).toBe( - 'sk-explicit', - ); - }); - - it('returns null on loopback without a credential (no synthetic bearer)', () => { - // Unlike LM Studio there is no documented default token, so we - // simply omit Authorization on loopback when no key is configured. - expect(selectAuthKey(null, 'http://localhost:8080/v1/chat/completions')).toBeNull(); - expect(selectAuthKey(null, 'http://127.0.0.1:8080/v1/chat/completions')).toBeNull(); - }); - - it('returns null on non-loopback without a credential', () => { - expect(selectAuthKey(null, 'https://my-tunnel.ngrok.io/v1/chat/completions')).toBeNull(); - expect(selectAuthKey(null, 'https://api.example.com/v1/chat/completions')).toBeNull(); - }); -}); - -describe('buildAuthHeaders', () => { - function sdkWith(cred: Credential | null): ISdk { - return { - trigger: vi.fn().mockResolvedValue(resolveResult(cred)), - registerFunction: vi.fn(), - } as unknown as ISdk; - } - - it('OMITS Authorization on loopback without a credential', async () => { - const headers = await buildAuthHeaders( - sdkWith(null), - 'http://localhost:8080/v1/chat/completions', - ); - expect('Authorization' in headers).toBe(false); - expect(headers['content-type']).toBe('application/json'); - }); - - it('OMITS Authorization on non-loopback without a credential', async () => { - const headers = await buildAuthHeaders( - sdkWith(null), - 'https://my-tunnel.ngrok.io/v1/chat/completions', - ); - expect('Authorization' in headers).toBe(false); - }); - - it('emits an explicit key when configured', async () => { - const headers = await buildAuthHeaders( - sdkWith({ type: 'api_key', key: 'sk-real' }), - 'http://localhost:8080/v1/chat/completions', - ); - expect(headers.Authorization).toBe('Bearer sk-real'); - }); -}); diff --git a/harness/tests/provider-llamacpp/config.test.ts b/harness/tests/provider-llamacpp/config.test.ts deleted file mode 100644 index d6cb93a72..000000000 --- a/harness/tests/provider-llamacpp/config.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { DEFAULT_API_URL, loadWorkerConfig } from '../../src/provider-llamacpp/config.js'; - -describe('loadWorkerConfig (llamacpp)', () => { - const original = process.env.LLAMACPP_BASE_URL; - - beforeEach(() => { - delete process.env.LLAMACPP_BASE_URL; - }); - - afterEach(() => { - if (original === undefined) delete process.env.LLAMACPP_BASE_URL; - else process.env.LLAMACPP_BASE_URL = original; - }); - - it('uses the localhost default (port 8080) when no yaml value and no env var', () => { - const cfg = loadWorkerConfig({}); - expect(cfg.default_api_url).toBe(DEFAULT_API_URL); - expect(cfg.default_api_url).toContain(':8080'); - expect(cfg.default_max_tokens).toBe(8192); - }); - - it('uses the yaml value when no env var is set', () => { - const cfg = loadWorkerConfig({ - provider_llamacpp: { - default_api_url: 'http://192.168.1.10:8080/v1/chat/completions', - default_max_tokens: 4096, - }, - }); - expect(cfg.default_api_url).toBe('http://192.168.1.10:8080/v1/chat/completions'); - expect(cfg.default_max_tokens).toBe(4096); - }); - - it('LLAMACPP_BASE_URL env var overrides the yaml value (full URL form)', () => { - process.env.LLAMACPP_BASE_URL = 'http://192.168.1.206:8080/v1/chat/completions'; - const cfg = loadWorkerConfig({ - provider_llamacpp: { default_api_url: DEFAULT_API_URL }, - }); - expect(cfg.default_api_url).toBe('http://192.168.1.206:8080/v1/chat/completions'); - }); - - it('LLAMACPP_BASE_URL appends /v1/chat/completions when given just a base origin', () => { - process.env.LLAMACPP_BASE_URL = 'http://192.168.1.206:8080'; - const cfg = loadWorkerConfig({}); - expect(cfg.default_api_url).toBe('http://192.168.1.206:8080/v1/chat/completions'); - }); - - it('LLAMACPP_BASE_URL trims a trailing slash before appending the path', () => { - process.env.LLAMACPP_BASE_URL = 'http://lan-host:8080/'; - const cfg = loadWorkerConfig({}); - expect(cfg.default_api_url).toBe('http://lan-host:8080/v1/chat/completions'); - }); - - it('LLAMACPP_BASE_URL empty/whitespace falls through to yaml/default', () => { - process.env.LLAMACPP_BASE_URL = ' '; - const cfg = loadWorkerConfig({ - provider_llamacpp: { default_api_url: 'http://yaml-host:8080/v1/chat/completions' }, - }); - expect(cfg.default_api_url).toBe('http://yaml-host:8080/v1/chat/completions'); - }); - - it('rejects malformed LLAMACPP_BASE_URL and falls through to yaml', () => { - process.env.LLAMACPP_BASE_URL = 'not a url at all'; - const cfg = loadWorkerConfig({ - provider_llamacpp: { default_api_url: 'http://yaml-host:8080/v1/chat/completions' }, - }); - expect(cfg.default_api_url).toBe('http://yaml-host:8080/v1/chat/completions'); - }); - - it('rejects non-http(s) schemes and falls through to yaml', () => { - process.env.LLAMACPP_BASE_URL = 'file:///etc/passwd'; - const cfg = loadWorkerConfig({ - provider_llamacpp: { default_api_url: 'http://yaml-host:8080/v1/chat/completions' }, - }); - expect(cfg.default_api_url).toBe('http://yaml-host:8080/v1/chat/completions'); - }); - - it('falls back to the localhost DEFAULT when BOTH env and yaml are malformed', () => { - process.env.LLAMACPP_BASE_URL = 'not-a-url'; - const cfg = loadWorkerConfig({ - provider_llamacpp: { default_api_url: 'also-not-a-url' }, - }); - expect(cfg.default_api_url).toBe(DEFAULT_API_URL); - }); -}); diff --git a/harness/tests/provider-llamacpp/discover.test.ts b/harness/tests/provider-llamacpp/discover.test.ts deleted file mode 100644 index fc5877fb5..000000000 --- a/harness/tests/provider-llamacpp/discover.test.ts +++ /dev/null @@ -1,283 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { - discoverAndRegister, - discoverLoadedModel, - modelsUrl, - propsUrl, - registerDiscovered, -} from '../../src/provider-llamacpp/discover.js'; -import type { ISdk } from '../../src/runtime/iii.js'; -import type { Model } from '../../src/models-catalog/types.js'; - -const originalFetch = globalThis.fetch; -afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); -}); - -describe('modelsUrl', () => { - it('derives /v1/models from the default chat URL', () => { - expect(modelsUrl('http://localhost:8080/v1/chat/completions')).toBe( - 'http://localhost:8080/v1/models', - ); - }); - - it('strips a trailing slash before appending', () => { - expect(modelsUrl('http://localhost:8080/v1/chat/completions/')).toBe( - 'http://localhost:8080/v1/models', - ); - }); - - it('falls through for non-canonical paths (appends `/models`)', () => { - expect(modelsUrl('http://localhost:8080/custom/path')).toBe( - 'http://localhost:8080/custom/path/models', - ); - }); -}); - -describe('propsUrl', () => { - it('derives /props at the server root from the default chat URL', () => { - expect(propsUrl('http://localhost:8080/v1/chat/completions')).toBe( - 'http://localhost:8080/props', - ); - }); - - it('strips a trailing slash on the chat URL before deriving /props', () => { - expect(propsUrl('http://localhost:8080/v1/chat/completions/')).toBe( - 'http://localhost:8080/props', - ); - }); - - it('strips a bare /v1 suffix', () => { - expect(propsUrl('http://localhost:8080/v1')).toBe('http://localhost:8080/props'); - }); - - it('appends /props to a custom-path proxy without /v1 (mirrors modelsUrl)', () => { - expect(propsUrl('http://localhost:8080/custom/path')).toBe( - 'http://localhost:8080/custom/path/props', - ); - }); -}); - -describe('discoverLoadedModel', () => { - it('returns one Model for the single loaded llama-server entry', async () => { - globalThis.fetch = vi.fn( - async () => - new Response(JSON.stringify({ data: [{ id: 'Meta-Llama-3.1-8B-Instruct' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }), - ) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out).toHaveLength(1); - expect(out[0]?.id).toBe('Meta-Llama-3.1-8B-Instruct'); - expect(out[0]?.provider).toBe('llamacpp'); - expect(out[0]?.supports_tools).toBe(true); - }); - - it('returns empty on a non-2xx response (best-effort)', async () => { - globalThis.fetch = vi.fn( - async () => new Response('boom', { status: 502 }), - ) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out).toEqual([]); - }); - - it('returns empty when the response is malformed JSON', async () => { - globalThis.fetch = vi.fn( - async () => new Response('not-json', { status: 200 }), - ) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out).toEqual([]); - }); - - it('returns empty when fetch throws (network error)', async () => { - globalThis.fetch = vi.fn(async () => { - throw new Error('ECONNREFUSED'); - }) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out).toEqual([]); - }); - - it('skips entries without a valid id', async () => { - globalThis.fetch = vi.fn( - async () => - new Response(JSON.stringify({ data: [{ id: '' }, { foo: 'bar' }, { id: 'valid' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }), - ) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out.map((m) => m.id)).toEqual(['valid']); - }); - - it('populates context_window from /props.n_ctx when available', async () => { - globalThis.fetch = vi.fn(async (input: RequestInfo | URL) => { - const url = typeof input === 'string' ? input : input.toString(); - if (url.endsWith('/props')) { - return new Response(JSON.stringify({ n_ctx: 262_144 }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - } - return new Response(JSON.stringify({ data: [{ id: 'Qwen3-35B-A3B' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - }) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out).toHaveLength(1); - expect(out[0]?.context_window).toBe(262_144); - }); - - it('falls back to the default context_window when /props is unavailable', async () => { - globalThis.fetch = vi.fn(async (input: RequestInfo | URL) => { - const url = typeof input === 'string' ? input : input.toString(); - if (url.endsWith('/props')) { - return new Response('boom', { status: 502 }); - } - return new Response(JSON.stringify({ data: [{ id: 'Qwen3-35B-A3B' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - }) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out).toHaveLength(1); - expect(out[0]?.context_window).toBe(32_768); - }); - - it('reads context_window from /props.default_generation_settings.n_ctx (older llama-server)', async () => { - globalThis.fetch = vi.fn(async (input: RequestInfo | URL) => { - const url = typeof input === 'string' ? input : input.toString(); - if (url.endsWith('/props')) { - return new Response(JSON.stringify({ default_generation_settings: { n_ctx: 131_072 } }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - } - return new Response(JSON.stringify({ data: [{ id: 'Qwen3-35B-A3B' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - }) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out[0]?.context_window).toBe(131_072); - }); - - it('prefers top-level /props.n_ctx over the nested default_generation_settings value', async () => { - globalThis.fetch = vi.fn(async (input: RequestInfo | URL) => { - const url = typeof input === 'string' ? input : input.toString(); - if (url.endsWith('/props')) { - return new Response( - JSON.stringify({ - n_ctx: 262_144, - default_generation_settings: { n_ctx: 4096 }, - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ); - } - return new Response(JSON.stringify({ data: [{ id: 'Qwen3-35B-A3B' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - }) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out[0]?.context_window).toBe(262_144); - }); - - it('falls back to the default context_window when /props omits or invalidates n_ctx', async () => { - globalThis.fetch = vi.fn(async (input: RequestInfo | URL) => { - const url = typeof input === 'string' ? input : input.toString(); - if (url.endsWith('/props')) { - return new Response(JSON.stringify({ n_ctx: -1 }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - } - return new Response(JSON.stringify({ data: [{ id: 'Qwen3-35B-A3B' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - }) as typeof globalThis.fetch; - const out = await discoverLoadedModel('http://localhost:8080/v1/chat/completions', {}); - expect(out[0]?.context_window).toBe(32_768); - }); -}); - -describe('registerDiscovered', () => { - it('reconciles all models in one catalog write', async () => { - const trigger = vi.fn().mockResolvedValue({ ids: ['a', 'b'], count: 2 }); - const sdk = { trigger, registerFunction: vi.fn() } as unknown as ISdk; - const models: Model[] = [ - { - id: 'a', - provider: 'llamacpp', - api: 'openai-completions', - display_name: 'a', - context_window: 1, - }, - { - id: 'b', - provider: 'llamacpp', - api: 'openai-completions', - display_name: 'b', - context_window: 1, - }, - ]; - const out = await registerDiscovered(sdk, models); - expect(out.sort()).toEqual(['a', 'b']); - expect(trigger).toHaveBeenCalledWith( - expect.objectContaining({ function_id: 'models::reconcile' }), - ); - }); - - it('returns [] when reconcile fails', async () => { - const trigger = vi.fn().mockRejectedValue(new Error('boom')); - const sdk = { trigger, registerFunction: vi.fn() } as unknown as ISdk; - const models: Model[] = [ - { - id: 'a', - provider: 'llamacpp', - api: 'openai-completions', - display_name: 'a', - context_window: 1, - }, - ]; - const out = await registerDiscovered(sdk, models); - expect(out).toEqual([]); - }); -}); - -describe('discoverAndRegister', () => { - it('returns an empty list when no models are reported', async () => { - globalThis.fetch = vi.fn( - async () => - new Response(JSON.stringify({ data: [] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }), - ) as typeof globalThis.fetch; - const trigger = vi.fn(); - const sdk = { trigger, registerFunction: vi.fn() } as unknown as ISdk; - const out = await discoverAndRegister(sdk, 'http://localhost:8080/v1/chat/completions', {}); - expect(out).toEqual([]); - expect(trigger).not.toHaveBeenCalled(); - }); - - it('end-to-end: discovers and registers the loaded model id', async () => { - globalThis.fetch = vi.fn( - async () => - new Response(JSON.stringify({ data: [{ id: 'Meta-Llama-3.1-8B' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }), - ) as typeof globalThis.fetch; - const trigger = vi.fn().mockResolvedValue(undefined); - const sdk = { trigger, registerFunction: vi.fn() } as unknown as ISdk; - const out = await discoverAndRegister(sdk, 'http://localhost:8080/v1/chat/completions', {}); - expect(out).toEqual(['Meta-Llama-3.1-8B']); - expect(trigger).toHaveBeenCalledWith( - expect.objectContaining({ function_id: 'models::reconcile' }), - ); - }); -}); diff --git a/harness/tests/provider-llamacpp/sse.test.ts b/harness/tests/provider-llamacpp/sse.test.ts deleted file mode 100644 index 3f1003af7..000000000 --- a/harness/tests/provider-llamacpp/sse.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - classifyLlamacppError, - emptyPartial, - extractErrorMessage, - handleChunk, - mapFinishReason, - syntheticErrorEvent, -} from '../../src/provider-llamacpp/sse.js'; - -describe('mapFinishReason', () => { - it.each([ - ['stop', 'end'], - ['length', 'length'], - ['tool_calls', 'function_call'], - ['function_call', 'function_call'], - ['somethingelse', 'end'], - ])('%s → %s', (input, expected) => { - expect(mapFinishReason(input)).toBe(expected); - }); -}); - -describe('extractErrorMessage', () => { - it('returns null when no error field', () => { - expect(extractErrorMessage({ choices: [] })).toBeNull(); - }); - - it('handles `error` as a string', () => { - expect(extractErrorMessage({ error: 'bad' })).toBe('bad'); - }); - - it('handles nested `error.message`', () => { - expect(extractErrorMessage({ error: { message: 'rate-limited' } })).toBe('rate-limited'); - }); - - it('handles nested `error.detail`', () => { - expect(extractErrorMessage({ error: { detail: 'bad request' } })).toBe('bad request'); - }); -}); - -describe('classifyLlamacppError', () => { - it('maps 401/403 to auth_expired', () => { - expect(classifyLlamacppError('forbidden', 401)).toBe('auth_expired'); - expect(classifyLlamacppError('forbidden', 403)).toBe('auth_expired'); - }); - - it('maps 429 to rate_limited', () => { - expect(classifyLlamacppError('slow down', 429)).toBe('rate_limited'); - }); - - it('maps 5xx to transient', () => { - expect(classifyLlamacppError('boom', 502)).toBe('transient'); - }); - - it('detects context_overflow via message', () => { - expect(classifyLlamacppError('context length exceeded')).toBe('context_overflow'); - expect(classifyLlamacppError('n_ctx exceeded')).toBe('context_overflow'); - }); - - it('falls through to permanent', () => { - expect(classifyLlamacppError('something else')).toBe('permanent'); - }); -}); - -describe('syntheticErrorEvent', () => { - it('puts the message in error_message ONLY (content stays empty)', () => { - const ev = syntheticErrorEvent('boom', 'm', 'llamacpp', 'transient'); - expect(ev.type).toBe('error'); - if (ev.type !== 'error') throw new Error('unreachable'); - expect(ev.error.error_message).toBe('boom'); - expect(ev.error.content).toEqual([]); - expect(ev.error.error_kind).toBe('transient'); - }); -}); - -describe('handleChunk', () => { - it('emits text_start + text_delta on first content delta', () => { - const state = emptyPartial(); - const events = handleChunk({ choices: [{ delta: { content: 'Hi' } }] }, state, 'm', 'llamacpp'); - expect(events.map((e) => e.type)).toEqual(['text_start', 'text_delta']); - }); - - it('only emits text_start once across deltas', () => { - const state = emptyPartial(); - handleChunk({ choices: [{ delta: { content: 'A' } }] }, state, 'm', 'llamacpp'); - const events = handleChunk({ choices: [{ delta: { content: 'B' } }] }, state, 'm', 'llamacpp'); - expect(events.map((e) => e.type)).toEqual(['text_delta']); - }); - - it('emits thinking events on reasoning_content', () => { - const state = emptyPartial(); - const events = handleChunk( - { choices: [{ delta: { reasoning_content: 'planning' } }] }, - state, - 'm', - 'llamacpp', - ); - expect(events.map((e) => e.type)).toEqual(['thinking_start', 'thinking_delta']); - }); - - it('records finish_reason and stop_reason mapping', () => { - const state = emptyPartial(); - handleChunk({ choices: [{ finish_reason: 'length', delta: {} }] }, state, 'm', 'llamacpp'); - expect(state.saw_finish_reason).toBe(true); - expect(state.stop_reason).toBe('length'); - }); - - it('rejects attacker-controlled tool_calls indices (DoS guard)', () => { - const state = emptyPartial(); - handleChunk( - { - choices: [ - { - delta: { - tool_calls: [{ index: 1e9, id: 't1', function: { name: 'x' } }], - }, - }, - ], - }, - state, - 'm', - 'llamacpp', - ); - // The unbounded index is dropped — tool_calls length stays 0. - expect(state.tool_calls.length).toBe(0); - }); - - it('returns an error event when the chunk carries an `error` field', () => { - const state = emptyPartial(); - const events = handleChunk( - { error: { message: 'No user query found' } }, - state, - 'm', - 'llamacpp', - ); - expect(events).toHaveLength(1); - expect(events[0]?.type).toBe('error'); - expect(state.saw_finish_reason).toBe(true); - expect(state.stop_reason).toBe('error'); - }); -}); diff --git a/harness/tests/provider-llamacpp/stream.test.ts b/harness/tests/provider-llamacpp/stream.test.ts deleted file mode 100644 index f747ec974..000000000 --- a/harness/tests/provider-llamacpp/stream.test.ts +++ /dev/null @@ -1,222 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { collect, streamLlamacpp } from '../../src/provider-llamacpp/stream.js'; -import type { ChatCompletionsConfig } from '../../src/provider-llamacpp/types.js'; - -const cfg: ChatCompletionsConfig = { - url: 'http://localhost:8080/v1/chat/completions', - provider_name: 'llamacpp', - model: 'Meta-Llama-3.1-8B', - api_key: '', - max_tokens: 1024, -}; - -function sseStream(chunks: string[]): ReadableStream { - const enc = new TextEncoder(); - let i = 0; - return new ReadableStream({ - pull(c) { - if (i < chunks.length) c.enqueue(enc.encode(chunks[i++])); - else c.close(); - }, - }); -} - -function okResponse(body: ReadableStream): Response { - return new Response(body, { - status: 200, - headers: { 'content-type': 'text/event-stream' }, - }); -} - -const originalFetch = globalThis.fetch; -afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); -}); - -describe('streamLlamacpp', () => { - it('streams a single text response end-to-end', async () => { - globalThis.fetch = vi.fn(async () => - okResponse( - sseStream([ - 'data: {"choices":[{"delta":{"content":"Hel"}}]}\n\n', - 'data: {"choices":[{"delta":{"content":"lo"}}]}\n\n', - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]), - ), - ) as typeof globalThis.fetch; - const final = await collect( - streamLlamacpp({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('end'); - const text = final.content.find((c) => c.type === 'text'); - if (text?.type !== 'text') throw new Error('missing text block'); - expect(text.text).toBe('Hello'); - }); - - it('omits Authorization when api_key is empty', async () => { - let observedAuth: string | undefined; - globalThis.fetch = vi.fn(async (_url, init) => { - observedAuth = (init as RequestInit | undefined)?.headers - ? (init as { headers: Record }).headers.Authorization - : undefined; - return okResponse( - sseStream([ - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]), - ); - }) as typeof globalThis.fetch; - await collect(streamLlamacpp({ cfg, system_prompt: '', messages: [], tools: [] })); - expect(observedAuth).toBeUndefined(); - }); - - it('emits Authorization when api_key is set', async () => { - let observedAuth: string | undefined; - globalThis.fetch = vi.fn(async (_url, init) => { - observedAuth = (init as { headers: Record }).headers.Authorization; - return okResponse( - sseStream([ - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]), - ); - }) as typeof globalThis.fetch; - await collect( - streamLlamacpp({ - cfg: { ...cfg, api_key: 'sk-real' }, - system_prompt: '', - messages: [], - tools: [], - }), - ); - expect(observedAuth).toBe('Bearer sk-real'); - }); - - it('surfaces a clear error when llama-server returns a non-2xx body (truncated)', async () => { - globalThis.fetch = vi.fn( - async () => - new Response(`${'x'.repeat(2000)}`, { - status: 500, - headers: { 'content-type': 'text/html' }, - }), - ) as typeof globalThis.fetch; - const final = await collect( - streamLlamacpp({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('error'); - expect(final.error_kind).toBe('transient'); - // Length-capped to ~256 bytes (the truncation guard). - expect((final.error_message ?? '').length).toBeLessThanOrEqual(256); - }); - - it('surfaces a stop-mid-stream error when SSE closes without [DONE] or finish_reason', async () => { - globalThis.fetch = vi.fn(async () => - okResponse( - sseStream([ - 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n', - // server closes the body here — no [DONE], no finish_reason - ]), - ), - ) as typeof globalThis.fetch; - const final = await collect( - streamLlamacpp({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('error'); - expect(final.error_message).toMatch(/stream closed mid-response/); - }); - - it('surfaces a clear error when a 200 response has no SSE chunks (wrong URL / HTML page)', async () => { - globalThis.fetch = vi.fn( - async () => - new Response('dashboard', { - status: 200, - headers: { 'content-type': 'text/html' }, - }), - ) as typeof globalThis.fetch; - const final = await collect( - streamLlamacpp({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('error'); - expect(final.error_message).toMatch(/non-SSE body/); - }); - - it('aborts the fetch after the configured timeout when the URL is unreachable', async () => { - // Pin the timeout via env so the fake-timer advance is precise; - // resolveFetchTimeoutMs is read fresh inside streamLlamacpp. - const prev = process.env.LLAMACPP_FETCH_TIMEOUT_MS; - process.env.LLAMACPP_FETCH_TIMEOUT_MS = '30000'; - vi.useFakeTimers(); - let abortedFromController = false; - globalThis.fetch = vi.fn((_url, init) => { - const signal = (init as RequestInit | undefined)?.signal; - return new Promise((_resolve, reject) => { - signal?.addEventListener('abort', () => { - abortedFromController = true; - const err = new Error('aborted'); - (err as Error & { name: string }).name = 'AbortError'; - reject(err); - }); - }); - }) as typeof globalThis.fetch; - try { - const final$ = collect(streamLlamacpp({ cfg, system_prompt: '', messages: [], tools: [] })); - await vi.advanceTimersByTimeAsync(30_001); - const final = await final$; - expect(abortedFromController).toBe(true); - expect(final.stop_reason).toBe('error'); - expect(final.error_message).toMatch(/timed out after 30s/i); - } finally { - vi.useRealTimers(); - if (prev === undefined) delete process.env.LLAMACPP_FETCH_TIMEOUT_MS; - else process.env.LLAMACPP_FETCH_TIMEOUT_MS = prev; - } - }); - - it('honours LLAMACPP_FETCH_TIMEOUT_MS for the abort message', async () => { - const prev = process.env.LLAMACPP_FETCH_TIMEOUT_MS; - process.env.LLAMACPP_FETCH_TIMEOUT_MS = '60000'; - vi.useFakeTimers(); - globalThis.fetch = vi.fn((_url, init) => { - const signal = (init as RequestInit | undefined)?.signal; - return new Promise((_resolve, reject) => { - signal?.addEventListener('abort', () => { - const err = new Error('aborted'); - (err as Error & { name: string }).name = 'AbortError'; - reject(err); - }); - }); - }) as typeof globalThis.fetch; - try { - const final$ = collect(streamLlamacpp({ cfg, system_prompt: '', messages: [], tools: [] })); - await vi.advanceTimersByTimeAsync(60_001); - const final = await final$; - expect(final.error_message).toMatch(/timed out after 60s/i); - } finally { - vi.useRealTimers(); - if (prev === undefined) delete process.env.LLAMACPP_FETCH_TIMEOUT_MS; - else process.env.LLAMACPP_FETCH_TIMEOUT_MS = prev; - } - }); - - it('falls back to the 120s default when LLAMACPP_FETCH_TIMEOUT_MS is unset', async () => { - // resolveFetchTimeoutMs is the unit; verify the contract here so a - // future default change is caught. - const prev = process.env.LLAMACPP_FETCH_TIMEOUT_MS; - delete process.env.LLAMACPP_FETCH_TIMEOUT_MS; - try { - const { resolveFetchTimeoutMs } = await import('../../src/provider-llamacpp/stream.js'); - expect(resolveFetchTimeoutMs()).toBe(120_000); - process.env.LLAMACPP_FETCH_TIMEOUT_MS = 'not-a-number'; - expect(resolveFetchTimeoutMs()).toBe(120_000); - process.env.LLAMACPP_FETCH_TIMEOUT_MS = '500'; // below floor - expect(resolveFetchTimeoutMs()).toBe(120_000); - process.env.LLAMACPP_FETCH_TIMEOUT_MS = '45000'; - expect(resolveFetchTimeoutMs()).toBe(45_000); - } finally { - if (prev === undefined) delete process.env.LLAMACPP_FETCH_TIMEOUT_MS; - else process.env.LLAMACPP_FETCH_TIMEOUT_MS = prev; - } - }); -}); diff --git a/harness/tests/provider-llamacpp/wire-messages.test.ts b/harness/tests/provider-llamacpp/wire-messages.test.ts deleted file mode 100644 index 18686ba58..000000000 --- a/harness/tests/provider-llamacpp/wire-messages.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - PLACEHOLDER_USER_MESSAGE, - toOpenaiMessages, -} from '../../src/provider-llamacpp/wire-messages.js'; -import type { AgentMessage } from '../../src/types/agent-message.js'; - -describe('toOpenaiMessages (llamacpp)', () => { - it('passes through plain user → assistant turns verbatim', () => { - const out = toOpenaiMessages( - [ - { - role: 'user', - content: [{ type: 'text', text: 'hello' }], - timestamp: 0, - }, - ], - 'be helpful', - ) as Array>; - expect(out[0]).toEqual({ role: 'system', content: 'be helpful' }); - expect(out[1]).toEqual({ role: 'user', content: 'hello' }); - }); - - it('round-trips reasoning_content on assistant tool-call messages', () => { - const messages: AgentMessage[] = [ - { - role: 'user', - content: [{ type: 'text', text: 'q' }], - timestamp: 0, - }, - { - role: 'assistant', - content: [ - { type: 'thinking', text: 'thinking step' }, - { - type: 'function_call', - id: 'call_01', - function_id: 'fs::ls', - arguments: { path: '.' }, - }, - ], - stop_reason: 'function_call', - model: 'm', - provider: 'llamacpp', - timestamp: 0, - }, - ]; - const out = toOpenaiMessages(messages, '') as Array>; - const asst = out.find((m) => m.role === 'assistant') as Record; - expect(asst).toBeDefined(); - expect(asst.reasoning_content).toBe('thinking step'); - expect(Array.isArray(asst.tool_calls)).toBe(true); - }); - - it('emits tool messages with content + tool_call_id + is_error', () => { - const messages: AgentMessage[] = [ - { - role: 'function_result', - function_call_id: 'tc1', - function_id: 'fs::read', - content: [{ type: 'text', text: '...' }], - details: {}, - is_error: true, - timestamp: 0, - }, - ]; - const out = toOpenaiMessages(messages, '') as Array>; - const tool = out.find((m) => m.role === 'tool') as Record; - expect(tool.tool_call_id).toBe('tc1'); - expect(tool.is_error).toBe(true); - }); - - it('injects a placeholder user message when no user-role message is present', () => { - const messages: AgentMessage[] = [ - { - role: 'function_result', - function_call_id: 'tc1', - function_id: 'fs::read', - content: [{ type: 'text', text: 'orphan' }], - details: {}, - is_error: false, - timestamp: 0, - }, - ]; - const out = toOpenaiMessages(messages, '') as Array>; - const userRow = out.find((m) => m.role === 'user'); - expect(userRow).toBeDefined(); - expect((userRow as { content: string }).content).toBe(PLACEHOLDER_USER_MESSAGE); - }); - - it('dedupes function_results with the same tool_call_id (latest wins)', () => { - const mk = (id: string, text: string): AgentMessage => ({ - role: 'function_result', - function_call_id: id, - function_id: 'fs::read', - content: [{ type: 'text', text }], - details: {}, - is_error: false, - timestamp: 0, - }); - const out = toOpenaiMessages([mk('a', 'first'), mk('a', 'second')], '') as Array< - Record - >; - const tools = out.filter((m) => m.role === 'tool'); - expect(tools).toHaveLength(1); - expect(tools[0]?.content).toBe('second'); - }); - - it('preserves order of distinct tool_call_ids while deduping repeats', () => { - const mk = (id: string, text: string): AgentMessage => ({ - role: 'function_result', - function_call_id: id, - function_id: 'fs::read', - content: [{ type: 'text', text }], - details: {}, - is_error: false, - timestamp: 0, - }); - const out = toOpenaiMessages( - [mk('a', 'A1'), mk('b', 'B1'), mk('a', 'A2'), mk('c', 'C1'), mk('b', 'B2')], - '', - ) as Array>; - const tools = out.filter((m) => m.role === 'tool') as Array<{ - tool_call_id: string; - content: string; - }>; - expect(tools.map((t) => t.tool_call_id)).toEqual(['a', 'b', 'c']); - expect(tools[0]?.content).toBe('A2'); - expect(tools[1]?.content).toBe('B2'); - expect(tools[2]?.content).toBe('C1'); - }); -}); diff --git a/harness/tests/provider-llamacpp/wire-tools.test.ts b/harness/tests/provider-llamacpp/wire-tools.test.ts deleted file mode 100644 index 3a23a785b..000000000 --- a/harness/tests/provider-llamacpp/wire-tools.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { functionsToOpenai } from '../../src/provider-llamacpp/wire-tools.js'; - -describe('functionsToOpenai (llamacpp)', () => { - it('maps each AgentFunction into the OpenAI tool envelope', () => { - const out = functionsToOpenai([ - { - name: 'sandbox::ls', - description: 'list dir', - parameters: { type: 'object', properties: {} }, - }, - { - name: 'fs::read', - description: 'read file', - parameters: { type: 'object', properties: { path: { type: 'string' } } }, - }, - ]) as Array<{ type: string; function: { name: string; description: string } }>; - expect(out).toHaveLength(2); - expect(out[0]?.type).toBe('function'); - expect(out[0]?.function.name).toBe('sandbox::ls'); - expect(out[1]?.function.name).toBe('fs::read'); - }); - - it('preserves an empty input as an empty array', () => { - expect(functionsToOpenai([])).toEqual([]); - }); -}); diff --git a/harness/tests/provider-lmstudio/auth.test.ts b/harness/tests/provider-lmstudio/auth.test.ts deleted file mode 100644 index b92e7fa99..000000000 --- a/harness/tests/provider-lmstudio/auth.test.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { - buildAuthHeaders, - buildConfig, - isLoopbackUrl, - selectAuthKey, -} from '../../src/provider-lmstudio/auth.js'; -import type { WorkerConfig } from '../../src/provider-lmstudio/config.js'; -import type { ISdk } from '../../src/runtime/iii.js'; -import type { Credential } from '../../src/runtime/provider-resolve.js'; - -const worker: WorkerConfig = { - default_max_tokens: 8192, - default_api_url: 'http://localhost:1234/v1/chat/completions', -}; - -/** - * Wrap a credential into the `harness::provider::resolve` result shape the - * provider now consumes (credential + non-secret settings in one call). - */ -function resolveResult(cred: Credential | null) { - return { - configured: cred !== null, - source: cred ? 'stored' : null, - credential: cred, - api_url: null, - max_tokens: null, - }; -} - -/** - * Narrow ISdk stub used only by `buildConfig` tests. By design this only - * implements `trigger`/`registerFunction` — `buildConfig` doesn't touch - * any other SDK surface. The trigger impl stands in for - * `harness::provider::resolve`. - */ -function makeSdk(triggerImpl: (req: { function_id: string; payload: unknown }) => unknown): ISdk { - return { - trigger: vi.fn(triggerImpl), - registerFunction: vi.fn(), - } as unknown as ISdk; -} - -describe('buildConfig (lmstudio)', () => { - it('falls back to api_key="lm-studio" when no credential is stored', async () => { - const sdk = makeSdk(() => resolveResult(null)); - const cfg = await buildConfig(sdk, worker, 'qwen/qwen3-4b-2507'); - expect(cfg.api_key).toBe('lm-studio'); - expect(cfg.url).toBe(worker.default_api_url); - expect(cfg.provider_name).toBe('lmstudio'); - expect(cfg.model).toBe('qwen/qwen3-4b-2507'); - expect(cfg.max_tokens).toBe(worker.default_max_tokens); - }); - - it('falls back to api_key="lm-studio" when resolve throws', async () => { - const sdk = makeSdk(() => { - throw new Error('harness unreachable'); - }); - const cfg = await buildConfig(sdk, worker, 'qwen/qwen3-4b-2507'); - expect(cfg.api_key).toBe('lm-studio'); - }); - - it('falls back to api_key="lm-studio" when credential has empty key', async () => { - const sdk = makeSdk(() => resolveResult({ type: 'api_key', key: '' })); - const cfg = await buildConfig(sdk, worker, 'qwen/qwen3-4b-2507'); - expect(cfg.api_key).toBe('lm-studio'); - }); - - it('honours a real API key when one is configured on an authenticated deployment', async () => { - const sdk = makeSdk(() => resolveResult({ type: 'api_key', key: 'sk-real-key' })); - const cfg = await buildConfig(sdk, worker, 'qwen/qwen3-4b-2507'); - expect(cfg.api_key).toBe('sk-real-key'); - }); - - it('resolves via harness::provider::resolve with provider="lmstudio"', async () => { - const trigger = vi.fn().mockResolvedValue(resolveResult(null)); - const sdk = { trigger, registerFunction: vi.fn() } as unknown as ISdk; - await buildConfig(sdk, worker, 'qwen/qwen3-4b-2507'); - expect(trigger).toHaveBeenCalledWith( - expect.objectContaining({ - function_id: 'harness::provider::resolve', - payload: { provider: 'lmstudio' }, - }), - ); - }); -}); - -describe('isLoopbackUrl', () => { - it('accepts canonical loopback hosts', () => { - expect(isLoopbackUrl('http://localhost:1234/v1/chat/completions')).toBe(true); - expect(isLoopbackUrl('http://127.0.0.1:1234/v1/chat/completions')).toBe(true); - expect(isLoopbackUrl('http://127.0.0.5/v1/chat/completions')).toBe(true); - expect(isLoopbackUrl('http://[::1]/v1/chat/completions')).toBe(true); - expect(isLoopbackUrl('https://api.localhost/v1/chat/completions')).toBe(true); - }); - - it('rejects non-loopback hosts', () => { - expect(isLoopbackUrl('http://example.com/v1/chat/completions')).toBe(false); - expect(isLoopbackUrl('http://192.168.1.10/v1/chat/completions')).toBe(false); - expect(isLoopbackUrl('https://my-tunnel.ngrok.io/v1/chat/completions')).toBe(false); - }); - - it('rejects malformed URLs (fail closed)', () => { - expect(isLoopbackUrl('not-a-url')).toBe(false); - expect(isLoopbackUrl('')).toBe(false); - }); -}); - -describe('selectAuthKey', () => { - it('returns the explicit key when one is configured', () => { - expect(selectAuthKey({ type: 'api_key', key: 'sk-explicit' }, 'http://localhost:1234/')).toBe( - 'sk-explicit', - ); - expect(selectAuthKey({ type: 'api_key', key: 'sk-explicit' }, 'https://remote.example/')).toBe( - 'sk-explicit', - ); - }); - - it('returns the fallback for loopback when no credential is configured', () => { - expect(selectAuthKey(null, 'http://localhost:1234/v1/chat/completions')).toBe('lm-studio'); - expect(selectAuthKey(null, 'http://127.0.0.1:1234/v1/chat/completions')).toBe('lm-studio'); - }); - - it('returns null (omit Authorization) for non-loopback without a credential', () => { - expect(selectAuthKey(null, 'https://my-tunnel.ngrok.io/v1/chat/completions')).toBeNull(); - expect(selectAuthKey(null, 'https://api.example.com/v1/chat/completions')).toBeNull(); - }); -}); - -describe('buildAuthHeaders', () => { - function sdkWith(cred: Credential | null): ISdk { - return { - trigger: vi.fn().mockResolvedValue(resolveResult(cred)), - registerFunction: vi.fn(), - } as unknown as ISdk; - } - - it('emits Authorization for loopback even without a credential', async () => { - const headers = await buildAuthHeaders( - sdkWith(null), - 'http://localhost:1234/v1/chat/completions', - ); - expect(headers.Authorization).toBe('Bearer lm-studio'); - expect(headers['content-type']).toBe('application/json'); - }); - - it('OMITS Authorization for non-loopback without a credential', async () => { - const headers = await buildAuthHeaders( - sdkWith(null), - 'https://my-tunnel.ngrok.io/v1/chat/completions', - ); - expect('Authorization' in headers).toBe(false); - expect(headers['content-type']).toBe('application/json'); - }); - - it('emits an explicit key on non-loopback when configured', async () => { - const headers = await buildAuthHeaders( - sdkWith({ type: 'api_key', key: 'sk-real' }), - 'https://my-tunnel.ngrok.io/v1/chat/completions', - ); - expect(headers.Authorization).toBe('Bearer sk-real'); - }); -}); diff --git a/harness/tests/provider-lmstudio/config.test.ts b/harness/tests/provider-lmstudio/config.test.ts deleted file mode 100644 index 16c1b949b..000000000 --- a/harness/tests/provider-lmstudio/config.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { DEFAULT_API_URL, loadWorkerConfig } from '../../src/provider-lmstudio/config.js'; - -describe('loadWorkerConfig (lmstudio)', () => { - const original = process.env.LMSTUDIO_BASE_URL; - - beforeEach(() => { - delete process.env.LMSTUDIO_BASE_URL; - }); - - afterEach(() => { - if (original === undefined) delete process.env.LMSTUDIO_BASE_URL; - else process.env.LMSTUDIO_BASE_URL = original; - }); - - it('uses the localhost default when no yaml value and no env var', () => { - const cfg = loadWorkerConfig({}); - expect(cfg.default_api_url).toBe(DEFAULT_API_URL); - expect(cfg.default_max_tokens).toBe(8192); - }); - - it('uses the yaml value when no env var is set', () => { - const cfg = loadWorkerConfig({ - provider_lmstudio: { - default_api_url: 'http://192.168.1.10:1234/v1/chat/completions', - default_max_tokens: 4096, - }, - }); - expect(cfg.default_api_url).toBe('http://192.168.1.10:1234/v1/chat/completions'); - expect(cfg.default_max_tokens).toBe(4096); - }); - - it('LMSTUDIO_BASE_URL env var overrides the yaml value (full URL form)', () => { - process.env.LMSTUDIO_BASE_URL = 'http://192.168.1.206:1234/v1/chat/completions'; - const cfg = loadWorkerConfig({ - provider_lmstudio: { default_api_url: DEFAULT_API_URL }, - }); - expect(cfg.default_api_url).toBe('http://192.168.1.206:1234/v1/chat/completions'); - }); - - it('LMSTUDIO_BASE_URL appends /v1/chat/completions when given just a base origin', () => { - process.env.LMSTUDIO_BASE_URL = 'http://192.168.1.206:1234'; - const cfg = loadWorkerConfig({}); - expect(cfg.default_api_url).toBe('http://192.168.1.206:1234/v1/chat/completions'); - }); - - it('LMSTUDIO_BASE_URL trims a trailing slash before appending the path', () => { - process.env.LMSTUDIO_BASE_URL = 'http://lan-host:1234/'; - const cfg = loadWorkerConfig({}); - expect(cfg.default_api_url).toBe('http://lan-host:1234/v1/chat/completions'); - }); - - it('LMSTUDIO_BASE_URL empty/whitespace falls through to yaml/default', () => { - process.env.LMSTUDIO_BASE_URL = ' '; - const cfg = loadWorkerConfig({ - provider_lmstudio: { default_api_url: 'http://yaml-host:5678/v1/chat/completions' }, - }); - expect(cfg.default_api_url).toBe('http://yaml-host:5678/v1/chat/completions'); - }); - - it('rejects malformed LMSTUDIO_BASE_URL and falls through to yaml', () => { - process.env.LMSTUDIO_BASE_URL = 'not a url at all'; - const cfg = loadWorkerConfig({ - provider_lmstudio: { default_api_url: 'http://yaml-host:5678/v1/chat/completions' }, - }); - expect(cfg.default_api_url).toBe('http://yaml-host:5678/v1/chat/completions'); - }); - - it('rejects non-http(s) schemes and falls through to yaml', () => { - process.env.LMSTUDIO_BASE_URL = 'file:///etc/passwd'; - const cfg = loadWorkerConfig({ - provider_lmstudio: { default_api_url: 'http://yaml-host:5678/v1/chat/completions' }, - }); - expect(cfg.default_api_url).toBe('http://yaml-host:5678/v1/chat/completions'); - }); - - it('falls back to the localhost DEFAULT when BOTH env and yaml are malformed', () => { - process.env.LMSTUDIO_BASE_URL = 'not-a-url'; - const cfg = loadWorkerConfig({ - provider_lmstudio: { default_api_url: 'also-not-a-url' }, - }); - expect(cfg.default_api_url).toBe(DEFAULT_API_URL); - }); -}); diff --git a/harness/tests/provider-lmstudio/discover.test.ts b/harness/tests/provider-lmstudio/discover.test.ts deleted file mode 100644 index e23323faa..000000000 --- a/harness/tests/provider-lmstudio/discover.test.ts +++ /dev/null @@ -1,406 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - discoverAllDownloadedModels, - discoverAndRegister, - discoverLoadedIds, - discoverLoadedModels, - nativeModelsUrl, - registerDiscovered, - toCatalogModel, -} from '../../src/provider-lmstudio/discover.js'; -import type { Model } from '../../src/models-catalog/types.js'; -import type { ISdk } from '../../src/runtime/iii.js'; - -describe('nativeModelsUrl', () => { - it('rewrites the OpenAI-compatible /v1/chat/completions path to /api/v0/models', () => { - expect(nativeModelsUrl('http://localhost:1234/v1/chat/completions')).toBe( - 'http://localhost:1234/api/v0/models', - ); - }); - - it('rewrites the native /api/v0/chat/completions path to /api/v0/models', () => { - expect(nativeModelsUrl('http://localhost:1234/api/v0/chat/completions')).toBe( - 'http://localhost:1234/api/v0/models', - ); - }); - - it('tolerates a trailing slash on the chat-completions path', () => { - expect(nativeModelsUrl('http://lan:1234/v1/chat/completions/')).toBe( - 'http://lan:1234/api/v0/models', - ); - }); - - it('handles a future /api/v2/chat/completions path', () => { - // Forward-compat for whenever LM Studio bumps the native version. - expect(nativeModelsUrl('http://h:1234/api/v2/chat/completions')).toBe( - 'http://h:1234/api/v0/models', - ); - }); -}); - -describe('toCatalogModel', () => { - it('maps a typical loaded LLM entry to the iii catalog Model shape', () => { - const out = toCatalogModel({ - id: 'qwen/qwen3-4b-2507', - type: 'llm', - state: 'loaded', - arch: 'qwen3', - quantization: 'Q4_K_M', - max_context_length: 32768, - loaded_context_length: 8192, - }); - expect(out).toEqual({ - id: 'qwen/qwen3-4b-2507', - provider: 'lmstudio', - api: 'openai-completions', - display_name: 'qwen/qwen3-4b-2507', - context_window: 8192, // prefers loaded over max - max_output_tokens: 8192, - supports_thinking: false, - supports_xhigh: false, - supports_tools: true, - supports_vision: false, - supports_cache: false, - transports: ['sse'], - }); - }); - - it('marks vision-language entries as supports_vision: true', () => { - const out = toCatalogModel({ id: 'a/b', type: 'vlm', state: 'loaded' }); - expect(out?.supports_vision).toBe(true); - }); - - it('returns null for embeddings models so we never route chat to them', () => { - expect(toCatalogModel({ id: 'nomic-embed', type: 'embeddings', state: 'loaded' })).toBeNull(); - }); - - it('returns null for entries missing an id', () => { - expect(toCatalogModel({ type: 'llm', state: 'loaded' })).toBeNull(); - }); - - it('falls back to max_context_length when loaded_context_length is missing', () => { - const out = toCatalogModel({ - id: 'm', - type: 'llm', - state: 'loaded', - max_context_length: 16384, - }); - expect(out?.context_window).toBe(16384); - }); - - it('falls back to the default context window when both lengths are missing', () => { - const out = toCatalogModel({ id: 'm', type: 'llm', state: 'loaded' }); - expect(out?.context_window).toBe(32768); - }); - - it('accepts entries with no `type` field (older LM Studio builds)', () => { - const out = toCatalogModel({ id: 'legacy/model', state: 'loaded' }); - expect(out?.id).toBe('legacy/model'); - }); -}); - -describe('discoverLoadedModels', () => { - let originalFetch: typeof globalThis.fetch; - - beforeEach(() => { - originalFetch = globalThis.fetch; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - }); - - it('returns the catalog Model for each loaded LLM, skipping unloaded and non-llm entries', async () => { - globalThis.fetch = vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - data: [ - { id: 'a/llm-loaded', type: 'llm', state: 'loaded', max_context_length: 4096 }, - { id: 'b/llm-not-loaded', type: 'llm', state: 'not-loaded', max_context_length: 4096 }, - { id: 'c/embed', type: 'embeddings', state: 'loaded' }, - { id: 'd/vlm-loaded', type: 'vlm', state: 'loaded', loaded_context_length: 2048 }, - ], - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ), - ) as typeof globalThis.fetch; - - const out = await discoverLoadedModels('http://localhost:1234/v1/chat/completions', { - Authorization: 'Bearer lm-studio', - }); - expect(out.map((m) => m.id)).toEqual(['a/llm-loaded', 'd/vlm-loaded']); - expect(out[1]?.supports_vision).toBe(true); - }); - - it('returns [] when LM Studio is unreachable (fetch throws)', async () => { - globalThis.fetch = vi - .fn() - .mockRejectedValue(new Error('ECONNREFUSED 127.0.0.1:1234')) as typeof globalThis.fetch; - const out = await discoverLoadedModels('http://localhost:1234/v1/chat/completions', {}); - expect(out).toEqual([]); - }); - - it('returns [] when LM Studio returns a non-2xx status', async () => { - globalThis.fetch = vi - .fn() - .mockResolvedValue(new Response('nope', { status: 500 })) as typeof globalThis.fetch; - const out = await discoverLoadedModels('http://localhost:1234/v1/chat/completions', {}); - expect(out).toEqual([]); - }); - - it('returns [] when LM Studio returns a 200 with invalid JSON', async () => { - globalThis.fetch = vi.fn().mockResolvedValue( - new Response('dashboard', { - status: 200, - headers: { 'content-type': 'text/html' }, - }), - ) as typeof globalThis.fetch; - const out = await discoverLoadedModels('http://localhost:1234/v1/chat/completions', {}); - expect(out).toEqual([]); - }); - - it('hits the /api/v0/models endpoint derived from the chat URL', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue( - new Response(JSON.stringify({ data: [] }), { status: 200 }), - ) as typeof globalThis.fetch; - globalThis.fetch = fetchMock; - - await discoverLoadedModels('http://192.168.0.206:1234/v1/chat/completions', { - Authorization: 'Bearer secret', - }); - - expect(fetchMock).toHaveBeenCalledTimes(1); - const [url, init] = (fetchMock as ReturnType).mock.calls[0] as [ - string, - RequestInit, - ]; - expect(url).toBe('http://192.168.0.206:1234/api/v0/models'); - expect(init.method).toBe('GET'); - expect((init.headers as Record).Authorization).toBe('Bearer secret'); - }); - - it('aborts the fetch after the discovery timeout when LM Studio hangs', async () => { - let aborted = false; - globalThis.fetch = vi.fn((_url, init) => { - const signal = (init as RequestInit).signal; - return new Promise((_resolve, reject) => { - signal?.addEventListener('abort', () => { - aborted = true; - const err = new Error('aborted'); - (err as Error & { name: string }).name = 'AbortError'; - reject(err); - }); - }); - }) as typeof globalThis.fetch; - - const out = await discoverLoadedModels('http://10.0.0.1:1234/v1/chat/completions', {}); - expect(out).toEqual([]); - expect(aborted).toBe(true); - }, 15_000); -}); - -describe('registerDiscovered', () => { - function makeModel(id: string): Model { - return { - id, - provider: 'lmstudio', - api: 'openai-completions', - display_name: id, - context_window: 8192, - max_output_tokens: 8192, - supports_thinking: false, - supports_xhigh: false, - supports_tools: true, - supports_vision: false, - supports_cache: false, - transports: ['sse'], - }; - } - - it('reconciles all models in one catalog write', async () => { - const trigger = vi.fn().mockResolvedValue({ ids: ['a', 'b'], count: 2 }); - const iii = { trigger } as unknown as ISdk; - - const out = await registerDiscovered(iii, [makeModel('a'), makeModel('b')]); - expect(out).toEqual(['a', 'b']); - expect(trigger).toHaveBeenCalledTimes(1); - expect(trigger).toHaveBeenCalledWith( - expect.objectContaining({ - function_id: 'models::reconcile', - payload: expect.objectContaining({ - provider: 'lmstudio', - models: expect.arrayContaining([ - expect.objectContaining({ id: 'a' }), - expect.objectContaining({ id: 'b' }), - ]), - }), - }), - ); - }); - - it('returns [] when reconcile fails', async () => { - const iii = { - trigger: vi.fn().mockRejectedValue(new Error('state worker down')), - } as unknown as ISdk; - - const out = await registerDiscovered(iii, [makeModel('a'), makeModel('b')]); - expect(out).toEqual([]); - }); -}); - -describe('discoverAllDownloadedModels', () => { - let originalFetch: typeof globalThis.fetch; - - beforeEach(() => { - originalFetch = globalThis.fetch; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - }); - - it('returns every LLM/VLM regardless of state — loaded AND not-loaded', async () => { - globalThis.fetch = vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - data: [ - { id: 'a/llm-loaded', type: 'llm', state: 'loaded' }, - { id: 'b/llm-cold', type: 'llm', state: 'not-loaded' }, - { id: 'c/embed', type: 'embeddings', state: 'loaded' }, - { id: 'd/vlm', type: 'vlm', state: 'not-loaded' }, - ], - }), - { status: 200 }, - ), - ) as typeof globalThis.fetch; - - const out = await discoverAllDownloadedModels('http://localhost:1234/v1/chat/completions', {}); - expect(out.map((m) => m.id)).toEqual(['a/llm-loaded', 'b/llm-cold', 'd/vlm']); - }); - - it('returns [] when LM Studio is unreachable', async () => { - globalThis.fetch = vi - .fn() - .mockRejectedValue(new Error('ECONNREFUSED')) as typeof globalThis.fetch; - const out = await discoverAllDownloadedModels('http://localhost:1234/v1/chat/completions', {}); - expect(out).toEqual([]); - }); -}); - -describe('discoverLoadedIds', () => { - let originalFetch: typeof globalThis.fetch; - - beforeEach(() => { - originalFetch = globalThis.fetch; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - }); - - it('returns a Set of every currently-loaded model id', async () => { - globalThis.fetch = vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - data: [ - { id: 'a', type: 'llm', state: 'loaded' }, - { id: 'b', type: 'llm', state: 'not-loaded' }, - { id: 'c', type: 'llm', state: 'loaded' }, - ], - }), - { status: 200 }, - ), - ) as typeof globalThis.fetch; - - const out = await discoverLoadedIds('http://localhost:1234/v1/chat/completions', {}); - expect(out.has('a')).toBe(true); - expect(out.has('b')).toBe(false); - expect(out.has('c')).toBe(true); - expect(out.size).toBe(2); - }); - - it('returns an empty Set when nothing is loaded', async () => { - globalThis.fetch = vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - data: [{ id: 'a', type: 'llm', state: 'not-loaded' }], - }), - { status: 200 }, - ), - ) as typeof globalThis.fetch; - - const out = await discoverLoadedIds('http://localhost:1234/v1/chat/completions', {}); - expect(out.size).toBe(0); - }); - - it('returns an empty Set when LM Studio is unreachable', async () => { - globalThis.fetch = vi - .fn() - .mockRejectedValue(new Error('ECONNREFUSED')) as typeof globalThis.fetch; - const out = await discoverLoadedIds('http://localhost:1234/v1/chat/completions', {}); - expect(out.size).toBe(0); - }); -}); - -describe('discoverAndRegister', () => { - let originalFetch: typeof globalThis.fetch; - - beforeEach(() => { - originalFetch = globalThis.fetch; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - }); - - it('registers every downloaded model (loaded AND not-loaded) so the picker shows them all', async () => { - globalThis.fetch = vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - data: [ - { id: 'qwen/qwen3-4b-2507', type: 'llm', state: 'loaded' }, - { id: 'meta/llama3.2-3b', type: 'llm', state: 'not-loaded' }, - { id: 'nomic/embed', type: 'embeddings', state: 'loaded' }, // skipped: non-llm - ], - }), - { status: 200 }, - ), - ) as typeof globalThis.fetch; - - const iii = { - trigger: vi.fn().mockResolvedValue({ ok: true }), - } as unknown as ISdk; - - const out = await discoverAndRegister(iii, 'http://localhost:1234/v1/chat/completions', {}); - expect(out).toEqual(['qwen/qwen3-4b-2507', 'meta/llama3.2-3b']); - - const calls = (iii.trigger as ReturnType).mock.calls; - const reconcile = calls.filter( - (c) => (c[0] as { function_id: string }).function_id === 'models::reconcile', - ); - expect(reconcile).toHaveLength(1); - const payload = (reconcile[0][0] as { payload: { provider: string; models: { id: string }[] } }) - .payload; - expect(payload.provider).toBe('lmstudio'); - expect(payload.models.map((m) => m.id)).toEqual(['qwen/qwen3-4b-2507', 'meta/llama3.2-3b']); - }); - - it('returns [] and never calls the register bus when LM Studio is unreachable', async () => { - globalThis.fetch = vi - .fn() - .mockRejectedValue(new Error('ECONNREFUSED')) as typeof globalThis.fetch; - const iii = { - trigger: vi.fn().mockResolvedValue({ ok: true }), - } as unknown as ISdk; - - const out = await discoverAndRegister(iii, 'http://localhost:1234/v1/chat/completions', {}); - expect(out).toEqual([]); - expect((iii.trigger as ReturnType).mock.calls).toHaveLength(0); - }); -}); diff --git a/harness/tests/provider-lmstudio/load.test.ts b/harness/tests/provider-lmstudio/load.test.ts deleted file mode 100644 index eeedb51b9..000000000 --- a/harness/tests/provider-lmstudio/load.test.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - loadModel, - nativeLoadUrl, - nativeUnloadUrl, - unloadModel, -} from '../../src/provider-lmstudio/load.js'; - -describe('nativeLoadUrl', () => { - it('rewrites /v1/chat/completions to /api/v1/models/load', () => { - expect(nativeLoadUrl('http://localhost:1234/v1/chat/completions')).toBe( - 'http://localhost:1234/api/v1/models/load', - ); - }); - - it('rewrites /api/v0/chat/completions to /api/v1/models/load', () => { - expect(nativeLoadUrl('http://localhost:1234/api/v0/chat/completions')).toBe( - 'http://localhost:1234/api/v1/models/load', - ); - }); -}); - -describe('nativeUnloadUrl', () => { - it('rewrites /v1/chat/completions to /api/v1/models/unload', () => { - expect(nativeUnloadUrl('http://localhost:1234/v1/chat/completions')).toBe( - 'http://localhost:1234/api/v1/models/unload', - ); - }); -}); - -describe('loadModel', () => { - let originalFetch: typeof globalThis.fetch; - - beforeEach(() => { - originalFetch = globalThis.fetch; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - }); - - it('POSTs to /api/v1/models/load and returns the load result', async () => { - const fetchMock = vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - type: 'llm', - instance_id: 'qwen/qwen3-4b-2507', - load_time_seconds: 7.42, - status: 'loaded', - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ), - ); - globalThis.fetch = fetchMock as typeof globalThis.fetch; - - const out = await loadModel( - 'http://localhost:1234/v1/chat/completions', - { Authorization: 'Bearer lm-studio' }, - 'qwen/qwen3-4b-2507', - { context_length: 8192, flash_attention: true }, - ); - - expect(out.status).toBe('loaded'); - expect(out.instance_id).toBe('qwen/qwen3-4b-2507'); - expect(out.load_time_seconds).toBe(7.42); - - expect(fetchMock).toHaveBeenCalledTimes(1); - const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; - expect(url).toBe('http://localhost:1234/api/v1/models/load'); - expect(init.method).toBe('POST'); - const body = JSON.parse(init.body as string) as Record; - expect(body.model).toBe('qwen/qwen3-4b-2507'); - expect(body.context_length).toBe(8192); - expect(body.flash_attention).toBe(true); - }); - - it('throws with the LM Studio error body on non-2xx status', async () => { - globalThis.fetch = vi - .fn() - .mockResolvedValue( - new Response('model not downloaded', { status: 404 }), - ) as typeof globalThis.fetch; - - await expect( - loadModel('http://localhost:1234/v1/chat/completions', {}, 'missing/model'), - ).rejects.toThrow(/lmstudio load failed \(404\).*model not downloaded/); - }); - - it('attaches a 404-specific hint pointing at LM Studio 0.4+ requirement', async () => { - // Older LM Studio (pre-0.4) returns 404 on the native v1 endpoint. - // Operators need to know that, not just "404 from somewhere". - globalThis.fetch = vi - .fn() - .mockResolvedValue(new Response('Not Found', { status: 404 })) as typeof globalThis.fetch; - - await expect(loadModel('http://localhost:1234/v1/chat/completions', {}, 'm')).rejects.toThrow( - /LM Studio 0\.4\+ only/i, - ); - }); - - it('attaches a generic common-causes hint on 500 responses', async () => { - // Regression: the QA report against zai-org/glm-4.7-flash returned a - // 500 with `{"error":{"type":"model_load_failed",...}}`. The user - // needs a checklist (downloaded, template, VRAM, quantization), not - // a bare "Failed to load model." pass-through. - globalThis.fetch = vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - error: { type: 'model_load_failed', message: 'Failed to load model.' }, - }), - { status: 500 }, - ), - ) as typeof globalThis.fetch; - - await expect( - loadModel('http://localhost:1234/v1/chat/completions', {}, 'zai-org/glm-4.7-flash'), - ).rejects.toThrow(/common causes:.*VRAM.*quantization.*Developer.*panel/is); - }); - - it('throws a timeout error when the load fetch aborts', async () => { - let abortedFromController = false; - globalThis.fetch = vi.fn((_url, init) => { - const signal = (init as RequestInit).signal; - return new Promise((_resolve, reject) => { - signal?.addEventListener('abort', () => { - abortedFromController = true; - const err = new Error('aborted'); - (err as Error & { name: string }).name = 'AbortError'; - reject(err); - }); - }); - }) as typeof globalThis.fetch; - - // Trim the timeout to keep the test fast — the actual load timeout - // would be 120s; we just need to prove the abort path surfaces a - // clear timeout error rather than the raw AbortError. - vi.useFakeTimers(); - const promise = loadModel('http://localhost:1234/v1/chat/completions', {}, 'huge/model'); - // Eagerly attach a no-op catch so vitest's "unhandled rejection" - // detector doesn't trip during fake-timer advancement (the - // rejection lands before the `expect(...).rejects` await binds, - // and the same Promise reference reaches both consumers fine). - promise.catch(() => {}); - await vi.advanceTimersByTimeAsync(120_001); - // Assert the AbortController fired (not just that the promise - // rejected with a /timed out/ message) so a refactor that returns - // a timeout error WITHOUT actually aborting the fetch is caught. - await expect(promise).rejects.toThrow(/timed out/i); - expect(abortedFromController).toBe(true); - vi.useRealTimers(); - }); - - it('propagates non-abort transport errors verbatim', async () => { - globalThis.fetch = vi - .fn() - .mockRejectedValue(new Error('ECONNREFUSED 127.0.0.1:1234')) as typeof globalThis.fetch; - await expect(loadModel('http://localhost:1234/v1/chat/completions', {}, 'm')).rejects.toThrow( - /ECONNREFUSED/, - ); - }); -}); - -describe('unloadModel', () => { - let originalFetch: typeof globalThis.fetch; - - beforeEach(() => { - originalFetch = globalThis.fetch; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - }); - - it('POSTs to /api/v1/models/unload with the instance_id and returns the response', async () => { - const fetchMock = vi.fn().mockResolvedValue( - new Response(JSON.stringify({ instance_id: 'qwen/qwen3-4b-2507' }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }), - ); - globalThis.fetch = fetchMock as typeof globalThis.fetch; - - const out = await unloadModel( - 'http://localhost:1234/v1/chat/completions', - {}, - 'qwen/qwen3-4b-2507', - ); - expect(out.instance_id).toBe('qwen/qwen3-4b-2507'); - - const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; - expect(url).toBe('http://localhost:1234/api/v1/models/unload'); - expect(init.method).toBe('POST'); - const body = JSON.parse(init.body as string) as Record; - expect(body.instance_id).toBe('qwen/qwen3-4b-2507'); - }); - - it('throws when LM Studio returns a non-2xx response', async () => { - globalThis.fetch = vi - .fn() - .mockResolvedValue(new Response('not loaded', { status: 400 })) as typeof globalThis.fetch; - await expect(unloadModel('http://localhost:1234/v1/chat/completions', {}, 'm')).rejects.toThrow( - /lmstudio unload failed \(400\)/, - ); - }); -}); diff --git a/harness/tests/provider-lmstudio/sse.test.ts b/harness/tests/provider-lmstudio/sse.test.ts deleted file mode 100644 index eeba9e23b..000000000 --- a/harness/tests/provider-lmstudio/sse.test.ts +++ /dev/null @@ -1,403 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - buildPartial, - classifyLmstudioError, - emptyPartial, - extractErrorMessage, - formatLmstudioError, - handleChunk, - isLoadFailureMessage, - mapFinishReason, - mergeUsage, - syntheticErrorEvent, -} from '../../src/provider-lmstudio/sse.js'; - -describe('mergeUsage (lmstudio)', () => { - it('extracts chat-completions cached_tokens', () => { - const u = { input: 0, output: 0, cache_read: 0, cache_write: 0 }; - mergeUsage( - { - prompt_tokens: 1500, - completion_tokens: 200, - prompt_tokens_details: { cached_tokens: 1200 }, - }, - u, - ); - expect(u.input).toBe(1500); - expect(u.output).toBe(200); - expect(u.cache_read).toBe(1200); - }); -}); - -describe('mapFinishReason (lmstudio)', () => { - it('maps known finish reasons', () => { - expect(mapFinishReason('stop')).toBe('end'); - expect(mapFinishReason('length')).toBe('length'); - expect(mapFinishReason('tool_calls')).toBe('function_call'); - expect(mapFinishReason('function_call')).toBe('function_call'); - }); - - it('falls back to `end` for unknown or empty finish reasons', () => { - // LM Studio GGUF backends sometimes emit non-standard finish reasons - // (e.g. content_filter from upstream OpenAI-compat layers). Pin the - // fallback so a future refactor doesn't accidentally introduce a new - // StopReason variant for unknown strings. - expect(mapFinishReason('content_filter')).toBe('end'); - expect(mapFinishReason('')).toBe('end'); - expect(mapFinishReason('unrecognised_string')).toBe('end'); - }); -}); - -describe('handleChunk (lmstudio)', () => { - it('emits text_start on first content delta then text_delta', () => { - const state = emptyPartial(); - const events = handleChunk( - { choices: [{ delta: { content: 'hi' } }] }, - state, - 'qwen/qwen3-4b-2507', - 'lmstudio', - ); - expect(events.map((e) => e.type)).toEqual(['text_start', 'text_delta']); - expect(state.text).toBe('hi'); - }); - - it('accumulates tool_call arguments across chunks', () => { - const state = emptyPartial(); - handleChunk( - { - choices: [ - { - delta: { - tool_calls: [ - { index: 0, id: 'tc1', function: { name: 'shell::exec', arguments: '{"x' } }, - ], - }, - }, - ], - }, - state, - 'qwen/qwen3-4b-2507', - 'lmstudio', - ); - handleChunk( - { - choices: [{ delta: { tool_calls: [{ index: 0, function: { arguments: '":1}' } }] } }], - }, - state, - 'qwen/qwen3-4b-2507', - 'lmstudio', - ); - expect(state.tool_calls[0]?.id).toBe('tc1'); - expect(state.tool_calls[0]?.function_id).toBe('shell::exec'); - expect(state.tool_calls[0]?.args_json).toBe('{"x":1}'); - }); - - it('records finish_reason as stop_reason on the partial state', () => { - const state = emptyPartial(); - handleChunk( - { choices: [{ finish_reason: 'tool_calls' }] }, - state, - 'qwen/qwen3-4b-2507', - 'lmstudio', - ); - const partial = buildPartial(state, 'qwen/qwen3-4b-2507', 'lmstudio'); - expect(partial.stop_reason).toBe('function_call'); - }); -}); - -describe('classifyLmstudioError', () => { - it('maps 401/403 to auth_expired (corporate proxy / authenticated deploy)', () => { - expect(classifyLmstudioError('unauthorized', 401)).toBe('auth_expired'); - expect(classifyLmstudioError('forbidden', 403)).toBe('auth_expired'); - }); - - it('maps 429 to rate_limited', () => { - expect(classifyLmstudioError('too many requests', 429)).toBe('rate_limited'); - }); - - it('maps 5xx to transient', () => { - expect(classifyLmstudioError('bad gateway', 502)).toBe('transient'); - }); - - it('maps "no model loaded" 4xx to transient', () => { - expect(classifyLmstudioError('No model is loaded. Please load a model first.', 404)).toBe( - 'transient', - ); - expect(classifyLmstudioError('model not found: foo/bar', 400)).toBe('transient'); - }); - - it('maps "model has crashed" 4xx to transient (post-JIT-load crash)', () => { - // Regression: this was the exact wire message zai-org/glm-4.7-flash - // emitted on lmstudio when its lazy load aborted. Previously - // classified as `permanent`, blocking any retry/recovery path. - expect( - classifyLmstudioError( - 'The model has crashed without additional information. (Exit code: null)', - 400, - ), - ).toBe('transient'); - }); - - it('maps model_load_failed type marker to transient', () => { - expect(classifyLmstudioError('{"type":"model_load_failed","message":"..."}', 400)).toBe( - 'transient', - ); - }); - - it('maps "Failed to load LLM" to transient', () => { - expect( - classifyLmstudioError(`Failed to load LLM 'foo/bar': Error: Failed to load model.`, 500), - ).toBe('transient'); - }); - - it('maps context length messages to context_overflow', () => { - expect(classifyLmstudioError('context length exceeded', 400)).toBe('context_overflow'); - }); - - it('defaults to permanent', () => { - expect(classifyLmstudioError('something else', 400)).toBe('permanent'); - }); -}); - -describe('isLoadFailureMessage', () => { - it('matches all known load-failure shapes', () => { - for (const msg of [ - 'No model is loaded', - 'model not found', - 'please load a model', - 'The model has crashed', - 'model_load_failed', - 'Failed to load LLM foo', - 'Failed to load model', - 'Exit code: null', - ]) { - expect(isLoadFailureMessage(msg), `expected hit for: ${msg}`).toBe(true); - } - }); - - it('does not match unrelated errors', () => { - for (const msg of [ - 'context length exceeded', - 'rate limited', - 'unauthorized', - 'template render failed', - ]) { - expect(isLoadFailureMessage(msg), `expected miss for: ${msg}`).toBe(false); - } - }); -}); - -describe('formatLmstudioError', () => { - it('prepends a load hint when the message looks like a load failure', () => { - const out = formatLmstudioError( - 'The model has crashed without additional information. (Exit code: null)', - ); - expect(out).toContain('LM Studio could not load the model'); - expect(out).toContain('provider::lmstudio::load_model'); - expect(out).toContain('pick a different model'); - // Original wire message must still be present after the hint. - expect(out).toContain('The model has crashed without additional information'); - }); - - it('passes through unrelated errors verbatim (no hint)', () => { - const msg = 'context length exceeded'; - expect(formatLmstudioError(msg)).toBe(msg); - }); -}); - -describe('syntheticErrorEvent applies the load-failure hint', () => { - it('puts the formatted message into error_message ONLY (not into content)', () => { - // Security-hardened contract: the formatted text now lives only in - // `error_message` (which the UI's translate layer surfaces via - // `stop-reason`). Pre-fix this same string was ALSO injected as a - // `text` ContentBlock — a malicious LM Studio backend or MITM - // could then smuggle "tool approved" lines or markup into the - // assistant content stream, where it would be persisted and - // re-fed to the next provider call as trusted context. Keeping - // content empty closes that prompt-injection sink. - const ev = syntheticErrorEvent( - 'The model has crashed without additional information. (Exit code: null)', - 'zai-org/glm-4.7-flash', - 'lmstudio', - 'transient', - ); - expect(ev.type).toBe('error'); - if (ev.type !== 'error') throw new Error('unreachable'); - const final = ev.error; - expect(final.error_message).toContain('LM Studio could not load the model'); - expect(final.error_message).toContain('Original error: The model has crashed'); - expect(final.content).toEqual([]); - }); - - it('leaves non-load-failure errors unmodified', () => { - const ev = syntheticErrorEvent('rate limited', 'm', 'lmstudio', 'rate_limited'); - if (ev.type !== 'error') throw new Error('unreachable'); - expect(ev.error.error_message).toBe('rate limited'); - expect(ev.error.content).toEqual([]); - }); -}); - -describe('extractErrorMessage', () => { - it('returns the inner message from an object-shaped error', () => { - expect( - extractErrorMessage({ - error: { message: 'No user query found in messages.', type: 'template_error' }, - }), - ).toBe('No user query found in messages.'); - }); - - it('returns a string-shaped error verbatim', () => { - expect(extractErrorMessage({ error: 'plain text error from server' })).toBe( - 'plain text error from server', - ); - }); - - it('falls back to .error.error_message when nested under that alias', () => { - expect(extractErrorMessage({ error: { error_message: 'OOM during generation' } })).toBe( - 'OOM during generation', - ); - }); - - it('falls back to .error.detail (used by some llama.cpp builds)', () => { - expect(extractErrorMessage({ error: { detail: 'context exceeded' } })).toBe('context exceeded'); - }); - - it('returns null when no error field is present', () => { - expect(extractErrorMessage({ choices: [{ delta: { content: 'hi' } }] })).toBeNull(); - }); - - it('returns null for empty/non-stringy error payloads', () => { - expect(extractErrorMessage({ error: '' })).toBeNull(); - expect(extractErrorMessage({ error: {} })).toBeNull(); - expect(extractErrorMessage({ error: null })).toBeNull(); - }); -}); - -describe('handleChunk — SSE error chunks', () => { - // LM Studio commits to HTTP 200 + SSE streaming, then on template render - // failure / mid-stream OOM / model unload it sends a JSON chunk shaped - // `data: {"error":{"message":"..."}}` with no `choices` and closes the - // body. Pre-fix we silently ignored those (no choices → no events) and - // the user saw "stream closed mid-response" instead of the real reason. - it('surfaces an error event when the chunk carries an object-shaped error', () => { - const state = emptyPartial(); - const events = handleChunk( - { - error: { - message: 'Error rendering prompt with jinja template: "No user query found in messages."', - type: 'invalid_request_error', - }, - }, - state, - 'qwen/qwen3.6-35b-a3b', - 'lmstudio', - ); - expect(events).toHaveLength(1); - expect(events[0]?.type).toBe('error'); - const err = events[0] as Extract<(typeof events)[number], { type: 'error' }>; - expect(err.error.stop_reason).toBe('error'); - expect(err.error.error_message).toMatch(/No user query found in messages/); - expect(state.stop_reason).toBe('error'); - expect(state.saw_finish_reason).toBe(true); - }); - - it('classifies "No user query" errors as transient (qwen template constraint, fixable)', () => { - const state = emptyPartial(); - const events = handleChunk( - { error: { message: 'No user query found in messages.' } }, - state, - 'qwen/qwen3-4b-2507', - 'lmstudio', - ); - const err = events[0] as Extract<(typeof events)[number], { type: 'error' }>; - // classifyLmstudioError doesn't have a specific regex for "No user - // query" — falls through to 'permanent'. That's fine: the UI's - // stop-reason surfacing still shows the real message. Pin the - // value (rather than `toBeDefined`) so a future reclassification - // (e.g. accidentally routing it through auth_expired) is caught. - expect(err.error.error_kind).toBe('permanent'); - }); - - it('classifies context-overflow error chunks as context_overflow', () => { - const state = emptyPartial(); - const events = handleChunk( - { error: { message: 'Trying to keep the first 8192 tokens, context length exceeded' } }, - state, - 'm', - 'lmstudio', - ); - const err = events[0] as Extract<(typeof events)[number], { type: 'error' }>; - expect(err.error.error_kind).toBe('context_overflow'); - }); - - it('continues normally when the chunk has choices and no error', () => { - const state = emptyPartial(); - const events = handleChunk({ choices: [{ delta: { content: 'hi' } }] }, state, 'm', 'lmstudio'); - expect(events.some((e) => e.type === 'error')).toBe(false); - expect(state.text).toBe('hi'); - }); -}); - -describe('handleChunk — thinking-mode reasoning_content (qwen3 / glm / deepseek-r1)', () => { - // LM Studio surfaces reasoning tokens for thinking-capable GGUFs on the - // standard OpenAI-compat `delta.reasoning_content` field — same shape - // as Moonshot Kimi K2. We persist them as a thinking ContentBlock so - // wire-messages can echo them back when the round-trip needs it. - - it('accumulates delta.reasoning_content and emits thinking_start + thinking_delta', () => { - const state = emptyPartial(); - const events = handleChunk( - { choices: [{ delta: { reasoning_content: 'reasoning…' } }] }, - state, - 'qwen/qwen3-thinking', - 'lmstudio', - ); - expect(state.reasoning_text).toBe('reasoning…'); - expect(events.map((e) => e.type)).toEqual(['thinking_start', 'thinking_delta']); - }); - - it('persists reasoning_text as a leading thinking ContentBlock', () => { - const state = emptyPartial(); - handleChunk( - { choices: [{ delta: { reasoning_content: 'planning' } }] }, - state, - 'm', - 'lmstudio', - ); - handleChunk({ choices: [{ delta: { content: 'answer' } }] }, state, 'm', 'lmstudio'); - const partial = buildPartial(state, 'm', 'lmstudio'); - expect(partial.content[0]).toEqual({ type: 'thinking', text: 'planning' }); - expect(partial.content[1]).toEqual({ type: 'text', text: 'answer' }); - }); - - it('carries the thinking block alongside tool_calls in one turn', () => { - const state = emptyPartial(); - handleChunk( - { - choices: [ - { - delta: { - reasoning_content: 'I should list files', - tool_calls: [ - { - index: 0, - id: 'tc-1', - function: { name: 'shell::ls', arguments: '{"path":"/"}' }, - }, - ], - }, - }, - ], - }, - state, - 'qwen/qwen3-thinking', - 'lmstudio', - ); - const partial = buildPartial(state, 'qwen/qwen3-thinking', 'lmstudio'); - const thinking = partial.content.find((c) => c.type === 'thinking') as - | { type: 'thinking'; text: string } - | undefined; - expect(thinking?.text).toBe('I should list files'); - expect(partial.content.some((c) => c.type === 'function_call')).toBe(true); - }); -}); diff --git a/harness/tests/provider-lmstudio/stream.test.ts b/harness/tests/provider-lmstudio/stream.test.ts deleted file mode 100644 index 84d702f4a..000000000 --- a/harness/tests/provider-lmstudio/stream.test.ts +++ /dev/null @@ -1,539 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { collect, streamLmstudio } from '../../src/provider-lmstudio/stream.js'; -import type { ChatCompletionsConfig } from '../../src/provider-lmstudio/types.js'; - -const cfg: ChatCompletionsConfig = { - url: 'http://localhost:1234/v1/chat/completions', - provider_name: 'lmstudio', - model: 'qwen/qwen3-4b-2507', - api_key: 'lm-studio', - max_tokens: 256, -}; - -function sseResponse(chunks: string[], status = 200): Response { - const encoder = new TextEncoder(); - const stream = new ReadableStream({ - start(controller) { - for (const c of chunks) controller.enqueue(encoder.encode(c)); - controller.close(); - }, - }); - return new Response(stream, { - status, - headers: { 'content-type': 'text/event-stream' }, - }); -} - -function errorResponse(status: number, body: string): Response { - return new Response(body, { status }); -} - -describe('streamLmstudio', () => { - let originalFetch: typeof globalThis.fetch; - - beforeEach(() => { - originalFetch = globalThis.fetch; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - }); - - it('emits start -> text_start -> text_delta+ -> done on a happy-path stream', async () => { - globalThis.fetch = vi - .fn() - .mockResolvedValue( - sseResponse([ - 'data: {"choices":[{"delta":{"content":"Hel"}}]}\n\n', - 'data: {"choices":[{"delta":{"content":"lo"}}]}\n\n', - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]), - ); - - const events: string[] = []; - let finalText = ''; - for await (const ev of streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] })) { - events.push(ev.type); - if (ev.type === 'done') { - finalText = ev.message.content - .filter((c): c is { type: 'text'; text: string } => c.type === 'text') - .map((c) => c.text) - .join(''); - } - } - expect(events).toEqual(['start', 'text_start', 'text_delta', 'text_delta', 'done']); - expect(finalText).toBe('Hello'); - }); - - it('classifies HTTP 502 (LM Studio not running) as transient', async () => { - globalThis.fetch = vi.fn().mockResolvedValue(errorResponse(502, 'connection refused upstream')); - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('error'); - expect(final.error_kind).toBe('transient'); - }); - - it('classifies "no model loaded" 4xx response as transient', async () => { - globalThis.fetch = vi - .fn() - .mockResolvedValue(errorResponse(404, 'no model is loaded; please load a model first')); - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.error_kind).toBe('transient'); - }); - - it('surfaces fetch transport failures (LM Studio offline) as a single error event', async () => { - globalThis.fetch = vi.fn().mockRejectedValue(new Error('ECONNREFUSED 127.0.0.1:1234')); - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('error'); - expect(final.error_message).toContain('lmstudio fetch failed'); - }); - - it('sends Bearer token and JSON body to the configured localhost URL', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue( - sseResponse([ - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]), - ); - globalThis.fetch = fetchMock; - - await collect(streamLmstudio({ cfg, system_prompt: 'sys', messages: [], tools: [] })); - - expect(fetchMock).toHaveBeenCalledTimes(1); - const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; - expect(url).toBe(cfg.url); - const headers = init.headers as Record; - expect(headers.Authorization).toBe('Bearer lm-studio'); - expect(headers['content-type']).toBe('application/json'); - const body = JSON.parse(init.body as string) as Record; - expect(body.model).toBe(cfg.model); - expect(body.stream).toBe(true); - expect((body.messages as Array<{ role: string }>)[0]?.role).toBe('system'); - }); - - it('emits an explicit error when LM Studio returns a 200 with a non-SSE body', async () => { - // LM Studio's UI dashboard at the wrong endpoint, or some builds when no - // model is loaded, return 200 + HTML. The stream must NOT silently - // succeed with an empty assistant message — emit a clear error event. - globalThis.fetch = vi.fn().mockResolvedValue( - new Response('LM Studio dashboard', { - status: 200, - headers: { 'content-type': 'text/html' }, - }), - ); - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('error'); - expect(final.error_message).toMatch(/non-SSE body/i); - expect(final.error_kind).toBe('transient'); - }); - - it('emits an explicit error when the stream closes mid-response without a finish_reason or [DONE]', async () => { - // This is the silent-truncation bug we're guarding against: LM Studio - // sometimes ends the SSE body abruptly (GPU OOM, model unloaded, - // host disconnect, context exhausted during generation). Before the - // fix the stream silently emitted `done` with stop_reason='end' — - // the user saw a half-written reply and zero error indication. The - // fix promotes this to an explicit error event so the UI can show a - // clear "stream closed mid-response" notice. - globalThis.fetch = vi.fn().mockResolvedValue( - sseResponse([ - 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n', - // No [DONE], no finish_reason — body just ends here. - ]), - ); - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('error'); - expect(final.error_kind).toBe('transient'); - expect(final.error_message).toMatch(/stream closed mid-response/i); - }); - - it('surfaces an SSE error chunk verbatim instead of the generic "stream closed" message', async () => { - // LM Studio commits to HTTP 200 + SSE, then on a prompt-template - // render failure (e.g. qwen3 "No user query found") sends a single - // chunk shaped `data: {"error":{"message":"..."}}` and closes. - // Without the fix that chunk's error field is silently ignored and - // the EOF guard reports the generic "stream closed mid-response". - // With the fix we surface LM Studio's specific message instead. - globalThis.fetch = vi - .fn() - .mockResolvedValue( - sseResponse([ - 'data: {"error":{"message":"Error rendering prompt with jinja template: \\"No user query found in messages.\\"","type":"invalid_request_error"}}\n\n', - ]), - ); - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('error'); - expect(final.error_message).toMatch(/No user query found in messages/); - // CRITICAL: must NOT be the generic fallback message. - expect(final.error_message).not.toMatch(/stream closed mid-response/i); - }); - - it('still emits a clean `done` when the stream sends finish_reason but no trailing [DONE]', async () => { - // Some servers omit the trailing `data: [DONE]\n\n` line but DO send - // a chunk with finish_reason. That's a legitimate end — we should - // NOT promote it to an error; the server explicitly said "stop". - globalThis.fetch = vi.fn().mockResolvedValue( - sseResponse([ - 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\n', - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - // No [DONE] sentinel after — body just ends. - ]), - ); - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('end'); - expect(final.error_message).toBeNull(); - }); - - it('aborts the fetch after the configured timeout when the URL is unreachable', async () => { - // Simulate fetch hanging indefinitely — the AbortController should - // fire and we should emit a clear timeout error, not hang the worker. - // Fake timers so we can advance past the production FETCH_TIMEOUT_MS - // (30s) in milliseconds rather than seconds of wall-clock. - vi.useFakeTimers(); - let abortedFromController = false; - globalThis.fetch = vi.fn((_url, init) => { - const signal = (init as RequestInit | undefined)?.signal; - return new Promise((_resolve, reject) => { - signal?.addEventListener('abort', () => { - abortedFromController = true; - const err = new Error('aborted'); - (err as Error & { name: string }).name = 'AbortError'; - reject(err); - }); - }); - }) as typeof globalThis.fetch; - - try { - const final$ = collect(streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] })); - // FETCH_TIMEOUT_MS is 30_000 in stream.ts — advance just past it. - await vi.advanceTimersByTimeAsync(30_001); - const final = await final$; - expect(abortedFromController).toBe(true); - expect(final.stop_reason).toBe('error'); - expect(final.error_message).toMatch(/timed out/i); - } finally { - vi.useRealTimers(); - } - }); - - it('resolves placeholder model `lmstudio-local` by querying /v1/models', async () => { - const calls: string[] = []; - globalThis.fetch = vi.fn(async (url, init) => { - calls.push(String(url)); - if (String(url).endsWith('/v1/models')) { - return new Response(JSON.stringify({ data: [{ id: 'qwen/qwen3.6-35b-a3b' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - } - // Stream call — capture the body to verify the model id was substituted. - const body = JSON.parse((init as RequestInit).body as string) as { model: string }; - expect(body.model).toBe('qwen/qwen3.6-35b-a3b'); - return sseResponse([ - 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\n', - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]); - }) as typeof globalThis.fetch; - - const placeholderCfg = { ...cfg, model: 'lmstudio-local' }; - const final = await collect( - streamLmstudio({ cfg: placeholderCfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(calls.some((u) => u.endsWith('/v1/models'))).toBe(true); - expect(final.stop_reason).toBe('end'); - expect(final.model).toBe('qwen/qwen3.6-35b-a3b'); - }); - - it('keeps the placeholder model when /v1/models discovery returns empty data', async () => { - globalThis.fetch = vi.fn(async (url) => { - if (String(url).endsWith('/v1/models')) { - return new Response(JSON.stringify({ data: [] }), { status: 200 }); - } - return sseResponse([ - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]); - }) as typeof globalThis.fetch; - - const placeholderCfg = { ...cfg, model: 'lmstudio-local' }; - const final = await collect( - streamLmstudio({ cfg: placeholderCfg, system_prompt: '', messages: [], tools: [] }), - ); - // Discovery returned empty list — fall back to the placeholder unchanged. - expect(final.model).toBe('lmstudio-local'); - }); - - it('forwards tool_call argument deltas as functioncall_delta events', async () => { - globalThis.fetch = vi - .fn() - .mockResolvedValue( - sseResponse([ - 'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"tc1","function":{"name":"shell::ls","arguments":"{\\"p"}}]}}]}\n\n', - 'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ath\\":\\"/tmp\\"}"}}]}}]}\n\n', - 'data: {"choices":[{"finish_reason":"tool_calls","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]), - ); - - const seen: string[] = []; - for await (const ev of streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] })) { - seen.push(ev.type); - } - expect(seen).toContain('functioncall_delta'); - expect(seen[seen.length - 1]).toBe('done'); - }); -}); - -describe('streamLmstudio — auto-load retry on load-failure error', () => { - // Regression: in the QA report for `zai-org/glm-4.7-flash` on lmstudio, - // the chat trigger failed with an SSE error chunk: - // {"error":{"message":"The model has crashed without additional - // information. (Exit code: null)","type":"model_load_failed"}} - // The fix: detect that shape on the first attempt, call the native - // `/api/v1/models/load` endpoint to (re)load the model, and retry the - // chat completion exactly once. These tests pin the new behavior. - let originalFetch: typeof globalThis.fetch; - beforeEach(() => { - originalFetch = globalThis.fetch; - }); - afterEach(() => { - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - }); - - function sse(chunks: string[]): Response { - const encoder = new TextEncoder(); - return new Response( - new ReadableStream({ - start(controller) { - for (const c of chunks) controller.enqueue(encoder.encode(c)); - controller.close(); - }, - }), - { status: 200, headers: { 'content-type': 'text/event-stream' } }, - ); - } - - it('auto-loads then succeeds when first attempt returns 400 "model has crashed"', async () => { - const urls: string[] = []; - globalThis.fetch = vi.fn(async (url) => { - const u = String(url); - urls.push(u); - if ( - u.endsWith('/v1/chat/completions') && - urls.filter((x) => x.endsWith('/v1/chat/completions')).length === 1 - ) { - // First attempt: LM Studio returns the crash error as 400 body. - return new Response( - JSON.stringify({ - error: { - type: 'model_load_failed', - message: 'The model has crashed without additional information. (Exit code: null)', - }, - }), - { status: 400, headers: { 'content-type': 'application/json' } }, - ); - } - if (u.endsWith('/api/v1/models/load')) { - return new Response( - JSON.stringify({ - type: 'llm', - instance_id: 'abc-123', - load_time_seconds: 1.5, - status: 'loaded', - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ); - } - // Second attempt: real streamed answer. - return sse([ - 'data: {"choices":[{"delta":{"content":"v24"}}]}\n\n', - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]); - }) as typeof globalThis.fetch; - - const events: string[] = []; - let finalText = ''; - for await (const ev of streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] })) { - events.push(ev.type); - if (ev.type === 'done') { - finalText = ev.message.content - .filter((c): c is { type: 'text'; text: string } => c.type === 'text') - .map((c) => c.text) - .join(''); - } - } - // Downstream must NOT see the failed first attempt — the buffered - // `start` from the first try gets dropped, the synthetic error never - // leaves the provider, and the second attempt's full event sequence - // is what reaches the consumer. - expect(events).toEqual(['start', 'text_start', 'text_delta', 'done']); - expect(finalText).toBe('v24'); - // load was actually called between the two chat attempts - expect(urls.filter((u) => u.endsWith('/api/v1/models/load'))).toHaveLength(1); - expect(urls.filter((u) => u.endsWith('/v1/chat/completions'))).toHaveLength(2); - }); - - it('auto-loads then succeeds when first attempt streams an SSE error chunk', async () => { - // Different code path: HTTP 200 + SSE, then the error arrives as a - // single data chunk. This is what kimi/qwen also emit on - // template-render failures. - let chatCallCount = 0; - globalThis.fetch = vi.fn(async (url) => { - const u = String(url); - if (u.endsWith('/v1/chat/completions')) { - chatCallCount++; - if (chatCallCount === 1) { - return sse([ - 'data: {"error":{"message":"The model has crashed without additional information. (Exit code: null)","type":"model_load_failed"}}\n\n', - ]); - } - return sse([ - 'data: {"choices":[{"delta":{"content":"ok"}}]}\n\n', - 'data: {"choices":[{"finish_reason":"stop","delta":{}}]}\n\n', - 'data: [DONE]\n\n', - ]); - } - // /api/v1/models/load - return new Response( - JSON.stringify({ - type: 'llm', - instance_id: 'abc-123', - load_time_seconds: 0.5, - status: 'loaded', - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ); - }) as typeof globalThis.fetch; - - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(final.stop_reason).toBe('end'); - expect(final.error_message).toBeNull(); - expect(chatCallCount).toBe(2); - }); - - it('does NOT retry a second time — surfaces the error if the retry also fails', async () => { - // Guard against infinite loops: if both attempts fail with the same - // load-failure shape, the SECOND error is what reaches the consumer. - let chatCallCount = 0; - globalThis.fetch = vi.fn(async (url) => { - const u = String(url); - if (u.endsWith('/api/v1/models/load')) { - return new Response( - JSON.stringify({ - type: 'llm', - instance_id: 'abc-123', - load_time_seconds: 0.5, - status: 'loaded', - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ); - } - chatCallCount++; - return new Response( - JSON.stringify({ - error: { type: 'model_load_failed', message: 'Failed to load LLM' }, - }), - { status: 400, headers: { 'content-type': 'application/json' } }, - ); - }) as typeof globalThis.fetch; - - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(chatCallCount).toBe(2); - expect(final.stop_reason).toBe('error'); - expect(final.error_kind).toBe('transient'); - expect(final.error_message).toContain('LM Studio could not load the model'); - expect(final.error_message).toContain('Failed to load LLM'); - }); - - it('does NOT retry on non-load-failure errors (e.g. context_overflow)', async () => { - let chatCallCount = 0; - globalThis.fetch = vi.fn(async () => { - chatCallCount++; - return new Response(JSON.stringify({ error: { message: 'context length exceeded' } }), { - status: 400, - headers: { 'content-type': 'application/json' }, - }); - }) as typeof globalThis.fetch; - - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(chatCallCount).toBe(1); - expect(final.error_kind).toBe('context_overflow'); - }); - - it('does NOT retry after tokens have already streamed (no mid-stream do-over)', async () => { - // If LM Studio sent us tokens then the model crashed, retrying would - // require us to discard partial output — confusing UX. The retry only - // fires when the first attempt's first non-start event is the error. - let chatCallCount = 0; - globalThis.fetch = vi.fn(async (url) => { - const u = String(url); - if (u.endsWith('/v1/chat/completions')) { - chatCallCount++; - return sse([ - 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n', - 'data: {"error":{"message":"The model has crashed without additional information. (Exit code: null)"}}\n\n', - ]); - } - throw new Error(`unexpected url ${u}`); - }) as typeof globalThis.fetch; - - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(chatCallCount).toBe(1); - expect(final.stop_reason).toBe('error'); - expect(final.error_message).toContain('LM Studio could not load the model'); - }); - - it('surfaces auto-load failure cleanly if /api/v1/models/load itself fails', async () => { - // Older LM Studio (pre-0.4) returns 404 on /api/v1/models/load; we - // shouldn't pretend the chat succeeded. - let chatCallCount = 0; - globalThis.fetch = vi.fn(async (url) => { - const u = String(url); - if (u.endsWith('/api/v1/models/load')) { - return new Response('Not Found', { status: 404 }); - } - chatCallCount++; - return new Response( - JSON.stringify({ error: { type: 'model_load_failed', message: 'The model has crashed' } }), - { status: 400, headers: { 'content-type': 'application/json' } }, - ); - }) as typeof globalThis.fetch; - - const final = await collect( - streamLmstudio({ cfg, system_prompt: '', messages: [], tools: [] }), - ); - expect(chatCallCount).toBe(1); // never reached the retry chat call - expect(final.stop_reason).toBe('error'); - expect(final.error_kind).toBe('transient'); - expect(final.error_message).toContain('auto-load failed'); - expect(final.error_message).toContain(cfg.model); - }); -}); diff --git a/harness/tests/provider-lmstudio/wire-messages.test.ts b/harness/tests/provider-lmstudio/wire-messages.test.ts deleted file mode 100644 index 41157034e..000000000 --- a/harness/tests/provider-lmstudio/wire-messages.test.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - PLACEHOLDER_USER_MESSAGE, - toOpenaiMessages, -} from '../../src/provider-lmstudio/wire-messages.js'; -import type { AgentMessage } from '../../src/types/agent-message.js'; - -describe('toOpenaiMessages (lmstudio)', () => { - it('prepends system message when present', () => { - const out = toOpenaiMessages([], 'be helpful') as Array>; - expect(out[0]).toEqual({ role: 'system', content: 'be helpful' }); - }); - - it('encodes assistant tool_calls with stringified arguments', () => { - const msg: AgentMessage = { - role: 'assistant', - content: [ - { type: 'text', text: 'calling' }, - { - type: 'function_call', - id: 'tc1', - function_id: 'shell::fs::ls', - arguments: { path: '/tmp' }, - }, - ], - stop_reason: 'function_call', - model: 'qwen/qwen3-4b-2507', - provider: 'lmstudio', - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - expect(out[0]?.role).toBe('assistant'); - const tcs = (out[0] as { tool_calls: Array<{ function: { arguments: string } }> }).tool_calls; - expect(tcs[0]?.function.arguments).toBe('{"path":"/tmp"}'); - }); - - it('emits tool messages with content + tool_call_id + is_error', () => { - const out = toOpenaiMessages( - [ - { - role: 'function_result', - function_call_id: 'tc1', - function_id: 'read', - content: [{ type: 'text', text: 'ok' }], - details: { status: 'denied' }, - is_error: true, - timestamp: 0, - }, - ], - '', - ) as Array>; - expect(out[0]?.role).toBe('tool'); - expect(out[0]?.tool_call_id).toBe('tc1'); - expect(out[0]?.is_error).toBe(true); - expect((out[0]?.content as string).startsWith('[PERMISSION_DENIED]')).toBe(true); - }); - - it('joins user text content with newlines', () => { - const msg: AgentMessage = { - role: 'user', - content: [ - { type: 'text', text: 'one' }, - { type: 'text', text: 'two' }, - ], - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - expect(out[0]?.content).toBe('one\ntwo'); - }); -}); - -describe('toOpenaiMessages — placeholder injection for strict templates', () => { - // qwen3's jinja template aborts with "No user query found in messages." - // when zero `role: 'user'` messages are present. This happens in normal - // agentic loops after a tool cycle whose original user message got - // summarised away by async compaction. We inject a minimal "(continue)" - // user message so the request still renders. - - it('appends a placeholder user message when no user message is present (tool-call only history)', () => { - const assistantWithToolCall: AgentMessage = { - role: 'assistant', - content: [ - { - type: 'function_call', - id: 'tc1', - function_id: 'shell::ls', - arguments: { path: '/' }, - }, - ], - stop_reason: 'function_call', - model: 'qwen/qwen3.6-35b-a3b', - provider: 'lmstudio', - timestamp: 0, - }; - const toolResult: AgentMessage = { - role: 'function_result', - function_call_id: 'tc1', - function_id: 'shell::ls', - content: [{ type: 'text', text: 'home\nroot\n' }], - details: {}, - is_error: false, - timestamp: 0, - }; - const out = toOpenaiMessages([assistantWithToolCall, toolResult], 'be helpful') as Array< - Record - >; - // system + assistant(tool_call) + tool + placeholder user - expect(out).toHaveLength(4); - expect(out[3]).toEqual({ role: 'user', content: PLACEHOLDER_USER_MESSAGE }); - }); - - it('appends a placeholder when messages is entirely empty (post-summary, pre-replay race)', () => { - const out = toOpenaiMessages([], '') as Array>; - expect(out).toEqual([{ role: 'user', content: PLACEHOLDER_USER_MESSAGE }]); - }); - - it('does NOT append a placeholder when a user message is already present', () => { - const userMsg: AgentMessage = { - role: 'user', - content: [{ type: 'text', text: 'hi' }], - timestamp: 0, - }; - const out = toOpenaiMessages([userMsg], '') as Array>; - expect(out).toHaveLength(1); - expect(out[0]).toEqual({ role: 'user', content: 'hi' }); - }); - - it('does NOT count system_prompt as a user message — placeholder still appends if no real user msg follows', () => { - const assistantOnly: AgentMessage = { - role: 'assistant', - content: [{ type: 'text', text: 'reply' }], - stop_reason: 'end', - model: 'm', - provider: 'lmstudio', - timestamp: 0, - }; - const out = toOpenaiMessages([assistantOnly], 'be terse') as Array>; - expect(out).toHaveLength(3); - expect(out[0]?.role).toBe('system'); - expect(out[1]?.role).toBe('assistant'); - expect(out[2]).toEqual({ role: 'user', content: PLACEHOLDER_USER_MESSAGE }); - }); -}); - -describe('toOpenaiMessages (lmstudio) — round-tripping reasoning_content', () => { - // Mirror of the Kimi fix: thinking-mode GGUFs served by LM Studio - // (qwen3, glm, deepseek-r1) need `reasoning_content` echoed back on - // assistant tool-call messages or they fail with the same template - // error Kimi K2.6 produces. - - it('emits reasoning_content when the assistant has a thinking block + tool_calls', () => { - const msg: AgentMessage = { - role: 'assistant', - content: [ - { type: 'thinking', text: 'I will list /' }, - { - type: 'function_call', - id: 'tc-1', - function_id: 'shell::ls', - arguments: { path: '/' }, - }, - ], - stop_reason: 'function_call', - model: 'qwen/qwen3-thinking', - provider: 'lmstudio', - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - expect(out[0]?.reasoning_content).toBe('I will list /'); - expect(out[0]?.tool_calls).toBeDefined(); - }); - - it('omits reasoning_content for non-thinking models (no thinking block)', () => { - const msg: AgentMessage = { - role: 'assistant', - content: [{ type: 'text', text: 'hi' }], - stop_reason: 'end', - model: 'qwen/qwen3-4b-2507', - provider: 'lmstudio', - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - expect(out[0]?.reasoning_content).toBeUndefined(); - }); -}); - -describe('toOpenaiMessages (lmstudio) — boundary dedup of duplicate tool messages', () => { - // Same defense as the other providers — LM Studio's jinja templates - // vary by GGUF; some accept duplicate tool messages, some don't. - // Latest-wins replace keeps every template happy. - - const mkResult = (id: string, text: string): AgentMessage => ({ - role: 'function_result', - function_call_id: id, - function_id: 'shell::run', - content: [{ type: 'text', text }], - details: {}, - is_error: false, - timestamp: 0, - }); - - it('keeps exactly one tool message per tool_call_id (latest wins)', () => { - const out = toOpenaiMessages( - [mkResult('call_01', 'first'), mkResult('call_01', 'second')], - '', - ) as Array>; - const tools = out.filter((m) => m.role === 'tool'); - expect(tools).toHaveLength(1); - expect(tools[0]?.content).toBe('second'); - }); - - it('preserves order of distinct tool_call_ids while deduping repeats', () => { - // Ported from provider-anthropic — keeps the two implementations - // in lockstep so a future divergence (e.g. earliest-wins on one - // provider) is caught. - const out = toOpenaiMessages( - [ - mkResult('a', 'A1'), - mkResult('b', 'B1'), - mkResult('a', 'A2'), - mkResult('c', 'C1'), - mkResult('b', 'B2'), - ], - '', - ) as Array>; - const tools = out.filter((m) => m.role === 'tool') as Array<{ - tool_call_id: string; - content: string; - }>; - expect(tools.map((t) => t.tool_call_id)).toEqual(['a', 'b', 'c']); - expect(tools[0]?.content).toBe('A2'); - expect(tools[1]?.content).toBe('B2'); - expect(tools[2]?.content).toBe('C1'); - }); - - it('dedup is scoped to the pending batch (across-batch repeats stay separate for openai-format too)', () => { - // OpenAI/LM Studio's wire format is flat (no batched user-msg - // wrapper like Anthropic), so duplicates here just become a - // single tool entry regardless of an assistant gap. We still - // verify the assistant boundary doesn't merge tool entries from - // different turns into one row. - const msgs: AgentMessage[] = [ - mkResult('a', 'r1'), - { - role: 'assistant', - content: [{ type: 'text', text: 'thinking…' }], - stop_reason: 'end', - model: 'qwen/qwen3-4b-2507', - provider: 'lmstudio', - timestamp: 0, - }, - mkResult('a', 'r2'), - ]; - const out = toOpenaiMessages(msgs, '') as Array>; - const tools = out.filter((m) => m.role === 'tool') as Array<{ - tool_call_id: string; - content: string; - }>; - // Latest-wins: a single tool row carries the most recent body. - // The assistant message between them survives intact. - expect(tools).toHaveLength(1); - expect(tools[0]?.content).toBe('r2'); - expect(out.filter((m) => m.role === 'assistant')).toHaveLength(1); - }); -}); diff --git a/harness/tests/provider-lmstudio/wire-tools.test.ts b/harness/tests/provider-lmstudio/wire-tools.test.ts deleted file mode 100644 index 4ac2d29cf..000000000 --- a/harness/tests/provider-lmstudio/wire-tools.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { functionsToOpenai } from '../../src/provider-lmstudio/wire-tools.js'; -import type { AgentFunction } from '../../src/types/function.js'; - -describe('functionsToOpenai (lmstudio)', () => { - // wire-tools.ts is intentionally separate from the OpenAI/Kimi copies so - // LM Studio-specific tool-format extensions can land without coupling - // providers. Pinning the exact output shape catches accidental drift - // (e.g. adding a `strict` field, renaming `function` → `tool`, etc.). - it('maps each AgentFunction to OpenAI tool shape with name, description, parameters', () => { - const fns: AgentFunction[] = [ - { - name: 'shell::exec', - description: 'Run a shell command', - parameters: { type: 'object', properties: { cmd: { type: 'string' } } }, - }, - { - name: 'fs::read', - description: 'Read a file', - parameters: { type: 'object', properties: { path: { type: 'string' } } }, - }, - ]; - const out = functionsToOpenai(fns) as Array>; - expect(out).toHaveLength(2); - expect(out[0]).toEqual({ - type: 'function', - function: { - name: 'shell::exec', - description: 'Run a shell command', - parameters: { type: 'object', properties: { cmd: { type: 'string' } } }, - }, - }); - expect(out[1]?.type).toBe('function'); - expect((out[1] as { function: { name: string } }).function.name).toBe('fs::read'); - }); - - it('returns an empty array for empty input (no leftover wrappers)', () => { - expect(functionsToOpenai([])).toEqual([]); - }); -}); diff --git a/harness/tests/provider-openai/reasoning.test.ts b/harness/tests/provider-openai/reasoning.test.ts deleted file mode 100644 index f574d3baf..000000000 --- a/harness/tests/provider-openai/reasoning.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { isReasoningModel, reasoningEffortFor } from '../../src/provider-openai/reasoning.js'; - -describe('isReasoningModel', () => { - it.each([ - 'gpt-5', - 'gpt-5-mini', - 'gpt-5.2', - 'o1-preview', - 'o3-mini', - 'o4-mini', - ])('detects %s by id pattern', (id) => { - expect(isReasoningModel(id)).toBe(true); - }); - - it.each(['gpt-4o', 'gpt-4.1', 'chatgpt-4o-latest'])('rejects %s by id pattern', (id) => { - expect(isReasoningModel(id)).toBe(false); - }); - - it('catalog flag wins over the id pattern', () => { - expect(isReasoningModel('gpt-4o', true)).toBe(true); - expect(isReasoningModel('gpt-5', false)).toBe(false); - }); -}); - -describe('reasoningEffortFor', () => { - it('defaults to medium when no level is given', () => { - expect(reasoningEffortFor(undefined, 'gpt-5')).toBe('medium'); - expect(reasoningEffortFor(undefined, 'o3-mini')).toBe('medium'); - }); - - it('passes supported levels through', () => { - expect(reasoningEffortFor('high', 'gpt-5')).toBe('high'); - expect(reasoningEffortFor('low', 'gpt-5.1')).toBe('low'); - }); - - it('gpt-5-pro is clamped to high regardless of level', () => { - expect(reasoningEffortFor('low', 'gpt-5-pro')).toBe('high'); - expect(reasoningEffortFor(undefined, 'gpt-5-pro')).toBe('high'); - expect(reasoningEffortFor('xhigh', 'gpt-5-pro')).toBe('high'); - }); - - it('xhigh only on gpt-5.2+ families', () => { - expect(reasoningEffortFor('xhigh', 'gpt-5.2')).toBe('xhigh'); - expect(reasoningEffortFor('xhigh', 'gpt-5.3-codex')).toBe('xhigh'); - expect(reasoningEffortFor('xhigh', 'gpt-5.1')).toBe('high'); - expect(reasoningEffortFor('xhigh', 'gpt-5')).toBe('high'); - }); - - it("maps 'off' to the lowest supported effort", () => { - expect(reasoningEffortFor('off', 'gpt-5.1')).toBe('none'); - expect(reasoningEffortFor('off', 'gpt-5')).toBe('minimal'); - expect(reasoningEffortFor('off', 'o3')).toBe('low'); - }); - - it("maps 'max' to xhigh where supported", () => { - expect(reasoningEffortFor('max', 'gpt-5.2')).toBe('xhigh'); - expect(reasoningEffortFor('max', 'o3')).toBe('high'); - }); - - it('returns undefined for chat-tuned variants', () => { - expect(reasoningEffortFor('high', 'gpt-5-chat-latest')).toBeUndefined(); - expect(reasoningEffortFor(undefined, 'gpt-5.2-chat-latest')).toBeUndefined(); - }); - - it('returns undefined for non-reasoning families', () => { - expect(reasoningEffortFor('high', 'gpt-4o')).toBeUndefined(); - }); - - it.each([ - 'o1', - 'o1-mini', - 'o1-preview', - 'o1-pro', - ])('returns undefined for %s (o1 family rejects reasoning_effort)', (id) => { - expect(reasoningEffortFor(undefined, id)).toBeUndefined(); - expect(reasoningEffortFor('high', id)).toBeUndefined(); - }); - - it('falls back to medium for an unrecognized level on families that support it', () => { - expect(reasoningEffortFor('turbo', 'gpt-5')).toBe('medium'); - expect(reasoningEffortFor('turbo', 'o3')).toBe('medium'); - }); - - it('falls back to the only supported effort for an unrecognized level on single-effort families', () => { - expect(reasoningEffortFor('turbo', 'gpt-5-pro')).toBe('high'); - }); -}); diff --git a/harness/tests/provider-openai/sse.test.ts b/harness/tests/provider-openai/sse.test.ts deleted file mode 100644 index 533b847f4..000000000 --- a/harness/tests/provider-openai/sse.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - buildPartial, - emptyPartial, - handleChunk, - mapFinishReason, - mergeUsage, -} from '../../src/provider-openai/sse.js'; - -describe('mergeUsage', () => { - it('extracts chat-completions cached_tokens', () => { - const u = { input: 0, output: 0, cache_read: 0, cache_write: 0 }; - mergeUsage( - { - prompt_tokens: 1500, - completion_tokens: 200, - prompt_tokens_details: { cached_tokens: 1200 }, - }, - u, - ); - expect(u.input).toBe(1500); - expect(u.output).toBe(200); - expect(u.cache_read).toBe(1200); - }); - - it('extracts responses-API cached_tokens', () => { - const u = { input: 0, output: 0, cache_read: 0, cache_write: 0 }; - mergeUsage( - { - input_tokens: 2000, - output_tokens: 100, - input_tokens_details: { cached_tokens: 1700 }, - }, - u, - ); - expect(u.cache_read).toBe(1700); - }); -}); - -describe('mapFinishReason', () => { - it('maps known finish reasons', () => { - expect(mapFinishReason('stop')).toBe('end'); - expect(mapFinishReason('length')).toBe('length'); - expect(mapFinishReason('tool_calls')).toBe('function_call'); - expect(mapFinishReason('function_call')).toBe('function_call'); - }); -}); - -describe('handleChunk', () => { - it('emits text_start on first content delta then text_delta', () => { - const state = emptyPartial(); - const events = handleChunk( - { - choices: [{ delta: { content: 'hello' } }], - }, - state, - 'gpt-5', - 'openai', - ); - expect(events.map((e) => e.type)).toEqual(['text_start', 'text_delta']); - expect(state.text).toBe('hello'); - }); - - it('accumulates tool_call arguments across chunks', () => { - const state = emptyPartial(); - handleChunk( - { - choices: [ - { - delta: { - tool_calls: [ - { index: 0, id: 'tc1', function: { name: 'shell::exec', arguments: '{"x' } }, - ], - }, - }, - ], - }, - state, - 'gpt-5', - 'openai', - ); - handleChunk( - { - choices: [ - { - delta: { - tool_calls: [{ index: 0, function: { arguments: '":1}' } }], - }, - }, - ], - }, - state, - 'gpt-5', - 'openai', - ); - expect(state.tool_calls[0]?.id).toBe('tc1'); - expect(state.tool_calls[0]?.function_id).toBe('shell::exec'); - expect(state.tool_calls[0]?.args_json).toBe('{"x":1}'); - }); - - it('records finish_reason as stop_reason on the partial state', () => { - const state = emptyPartial(); - handleChunk({ choices: [{ finish_reason: 'tool_calls' }] }, state, 'gpt-5', 'openai'); - expect(state.stop_reason).toBe('function_call'); - const partial = buildPartial(state, 'gpt-5', 'openai'); - expect(partial.stop_reason).toBe('function_call'); - }); -}); diff --git a/harness/tests/provider-openai/stream-request.test.ts b/harness/tests/provider-openai/stream-request.test.ts deleted file mode 100644 index cfc75e4c8..000000000 --- a/harness/tests/provider-openai/stream-request.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { Model } from '../../src/models-catalog/types.js'; -import { streamOpenai } from '../../src/provider-openai/stream.js'; -import type { ChatCompletionsConfig } from '../../src/provider-openai/types.js'; - -function cfg(overrides: Partial = {}): ChatCompletionsConfig { - return { - url: 'https://api.example/v1/chat/completions', - provider_name: 'openai', - model: 'gpt-5', - api_key: 'sk-test', - max_tokens: 32_000, - ...overrides, - }; -} - -function catalogModel(extra: Partial = {}): Model { - return { - id: 'gpt-4o', - provider: 'openai', - api: 'openai-responses', - display_name: 'GPT-4o', - context_window: 128_000, - ...extra, - }; -} - -async function captureBody( - args: Parameters[0], -): Promise> { - let captured: Record | null = null; - vi.stubGlobal( - 'fetch', - vi.fn().mockImplementation(async (_url: string, init: RequestInit) => { - captured = JSON.parse(init.body as string) as Record; - return new Response('data: [DONE]\n\n', { status: 200 }); - }), - ); - for await (const _ev of streamOpenai(args)) { - // drain - } - if (!captured) throw new Error('fetch was not called'); - return captured; -} - -afterEach(() => { - vi.unstubAllGlobals(); -}); - -describe('streamOpenai request body', () => { - it('sends the resolved max_completion_tokens', async () => { - const body = await captureBody({ cfg: cfg(), system_prompt: 's', messages: [], tools: [] }); - expect(body.max_completion_tokens).toBe(32_000); - }); - - it('defaults reasoning_effort to medium for reasoning models', async () => { - const body = await captureBody({ cfg: cfg(), system_prompt: 's', messages: [], tools: [] }); - expect(body.reasoning_effort).toBe('medium'); - }); - - it('maps thinking_level onto reasoning_effort', async () => { - const body = await captureBody({ - cfg: cfg({ model: 'gpt-5.2' }), - system_prompt: 's', - messages: [], - tools: [], - thinking_level: 'xhigh', - }); - expect(body.reasoning_effort).toBe('xhigh'); - }); - - it('omits reasoning_effort for non-reasoning models', async () => { - const body = await captureBody({ - cfg: cfg({ model: 'gpt-4o' }), - system_prompt: 's', - messages: [], - tools: [], - }); - expect(body).not.toHaveProperty('reasoning_effort'); - }); - - it('catalog supports_thinking=true enables reasoning_effort for non-pattern ids', async () => { - const body = await captureBody({ - cfg: cfg({ model: 'o3-custom', catalog: catalogModel({ supports_thinking: true }) }), - system_prompt: 's', - messages: [], - tools: [], - }); - expect(body.reasoning_effort).toBe('medium'); - }); - - it('catalog supports_thinking=false disables reasoning_effort despite the id pattern', async () => { - const body = await captureBody({ - cfg: cfg({ model: 'gpt-5', catalog: catalogModel({ supports_thinking: false }) }), - system_prompt: 's', - messages: [], - tools: [], - }); - expect(body).not.toHaveProperty('reasoning_effort'); - }); -}); diff --git a/harness/tests/provider-openai/wire-messages.test.ts b/harness/tests/provider-openai/wire-messages.test.ts deleted file mode 100644 index 573b4de06..000000000 --- a/harness/tests/provider-openai/wire-messages.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { toOpenaiMessages } from '../../src/provider-openai/wire-messages.js'; -import type { AgentMessage } from '../../src/types/agent-message.js'; - -describe('toOpenaiMessages', () => { - it('prepends system message when present', () => { - const out = toOpenaiMessages([], 'be helpful') as Array>; - expect(out[0]).toEqual({ role: 'system', content: 'be helpful' }); - }); - - it('encodes assistant tool_calls with stringified arguments', () => { - const msg: AgentMessage = { - role: 'assistant', - content: [ - { type: 'text', text: 'calling' }, - { - type: 'function_call', - id: 'tc1', - function_id: 'shell::fs::ls', - arguments: { path: '/tmp' }, - }, - ], - stop_reason: 'function_call', - model: 'gpt-5', - provider: 'openai', - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - expect(out[0]?.role).toBe('assistant'); - const tcs = (out[0] as { tool_calls: Array<{ function: { arguments: string } }> }).tool_calls; - expect(tcs[0]?.function.arguments).toBe('{"path":"/tmp"}'); - }); - - it('emits tool messages with content + tool_call_id + is_error', () => { - const out = toOpenaiMessages( - [ - { - role: 'function_result', - function_call_id: 'tc1', - function_id: 'read', - content: [{ type: 'text', text: 'ok' }], - details: { status: 'denied' }, - is_error: true, - timestamp: 0, - }, - ], - '', - ) as Array>; - expect(out[0]?.role).toBe('tool'); - expect(out[0]?.tool_call_id).toBe('tc1'); - expect(out[0]?.is_error).toBe(true); - expect(typeof out[0]?.content).toBe('string'); - // denial envelope should be embedded as [PERMISSION_DENIED] prefix - expect((out[0]?.content as string).startsWith('[PERMISSION_DENIED]')).toBe(true); - }); - - it('joins user text content with newlines', () => { - const msg: AgentMessage = { - role: 'user', - content: [ - { type: 'text', text: 'one' }, - { type: 'text', text: 'two' }, - ], - timestamp: 0, - }; - const out = toOpenaiMessages([msg], '') as Array>; - expect(out[0]?.content).toBe('one\ntwo'); - }); - - describe('boundary dedup of duplicate tool messages', () => { - // OpenAI's wire shape emits one `{role:'tool', tool_call_id}` message - // per function_result (not a bundled content array like Anthropic). - // Without dedup, orchestrator re-entry would ship two tool messages - // with the same tool_call_id — some servers reject this, some silently - // overwrite. Dedup makes behavior deterministic regardless. - - const mkResult = (id: string, text: string): AgentMessage => ({ - role: 'function_result', - function_call_id: id, - function_id: 'shell::run', - content: [{ type: 'text', text }], - details: {}, - is_error: false, - timestamp: 0, - }); - - it('keeps exactly one tool message per tool_call_id (latest wins)', () => { - const out = toOpenaiMessages( - [mkResult('call_01', 'first'), mkResult('call_01', 'second')], - '', - ) as Array>; - const tools = out.filter((m) => m.role === 'tool'); - expect(tools).toHaveLength(1); - expect(tools[0]?.tool_call_id).toBe('call_01'); - expect(tools[0]?.content).toBe('second'); - }); - - it('preserves order of distinct tool_call_ids', () => { - const out = toOpenaiMessages( - [mkResult('a', 'A1'), mkResult('b', 'B1'), mkResult('a', 'A2'), mkResult('c', 'C1')], - '', - ) as Array>; - const tools = out.filter((m) => m.role === 'tool'); - expect(tools.map((t) => t.tool_call_id)).toEqual(['a', 'b', 'c']); - expect(tools[0]?.content).toBe('A2'); - }); - }); -}); diff --git a/harness/tests/runtime/models-discovery.test.ts b/harness/tests/runtime/models-discovery.test.ts deleted file mode 100644 index 909315ea7..000000000 --- a/harness/tests/runtime/models-discovery.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { - enrichModel, - fetchModelsForDiscovery, - fetchModelsJson, -} from '../../src/runtime/models-discovery.js'; - -describe('fetchModelsForDiscovery', () => { - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it('returns ok with parsed json on 200', async () => { - const body = { data: [{ id: 'm1' }] }; - globalThis.fetch = vi - .fn() - .mockResolvedValue( - new Response(JSON.stringify(body), { status: 200 }), - ) as typeof globalThis.fetch; - - const result = await fetchModelsForDiscovery('https://api.example/v1/models', {}); - expect(result).toEqual({ kind: 'ok', json: body }); - }); - - it('returns auth_error on 401 and 403', async () => { - for (const status of [401, 403] as const) { - globalThis.fetch = vi - .fn() - .mockResolvedValue(new Response('', { status })) as typeof globalThis.fetch; - const result = await fetchModelsForDiscovery('https://api.example/v1/models', {}); - expect(result).toEqual({ kind: 'auth_error', status }); - } - }); - - it('returns transient_error on 503', async () => { - globalThis.fetch = vi - .fn() - .mockResolvedValue(new Response('', { status: 503 })) as typeof globalThis.fetch; - const result = await fetchModelsForDiscovery('https://api.example/v1/models', {}); - expect(result).toEqual({ kind: 'transient_error', status: 503 }); - }); - - it('fetchModelsJson returns null for auth_error', async () => { - globalThis.fetch = vi - .fn() - .mockResolvedValue(new Response('', { status: 401 })) as typeof globalThis.fetch; - expect(await fetchModelsJson('https://api.example/v1/models', {})).toBeNull(); - }); -}); - -describe('enrichModel', () => { - const base = { - provider: 'anthropic', - api: 'anthropic-messages', - stub: { id: 'claude-sonnet-4-6' }, - defaultContextWindow: 200_000, - }; - - it('falls back to provider defaults without models.dev metadata', () => { - const m = enrichModel(base); - expect(m.context_window).toBe(200_000); - expect(m.max_output_tokens).toBe(8_192); - expect(m.supports_tools).toBe(true); - expect(m.supports_thinking).toBeUndefined(); - }); - - it('prefers models.dev limits and capability flags', () => { - const m = enrichModel({ - ...base, - modelsDev: { - id: 'claude-sonnet-4-6', - limit: { context: 200_000, output: 64_000 }, - reasoning: true, - tool_call: true, - }, - }); - expect(m.context_window).toBe(200_000); - expect(m.max_output_tokens).toBe(64_000); - expect(m.supports_thinking).toBe(true); - expect(m.supports_tools).toBe(true); - }); - - it('keeps defaults for fields models.dev omits', () => { - const m = enrichModel({ ...base, modelsDev: { id: 'claude-sonnet-4-6', reasoning: false } }); - expect(m.max_output_tokens).toBe(8_192); - expect(m.context_window).toBe(200_000); - expect(m.supports_thinking).toBe(false); - }); -}); diff --git a/harness/tests/runtime/modelsdev.test.ts b/harness/tests/runtime/modelsdev.test.ts deleted file mode 100644 index 4e3730a69..000000000 --- a/harness/tests/runtime/modelsdev.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { - _resetModelsDevForTests, - getModelsDevIndex, - lookupModelsDev, - MODELSDEV_URL, -} from '../../src/runtime/modelsdev.js'; - -const API_JSON = { - anthropic: { - id: 'anthropic', - models: { - 'claude-sonnet-4-6': { - id: 'claude-sonnet-4-6', - reasoning: true, - tool_call: true, - limit: { context: 200_000, output: 64_000 }, - }, - 'claude-sonnet-4': { - id: 'claude-sonnet-4', - reasoning: true, - tool_call: true, - limit: { context: 200_000, output: 64_000 }, - }, - }, - }, - openai: { - id: 'openai', - models: { - 'gpt-5-pro': { - id: 'gpt-5-pro', - reasoning: true, - tool_call: true, - limit: { context: 400_000, output: 272_000 }, - }, - }, - }, - moonshotai: { - id: 'moonshotai', - models: { - 'kimi-k2.6': { id: 'kimi-k2.6', reasoning: true, limit: { context: 256_000, output: 8_192 } }, - }, - }, -}; - -afterEach(() => { - vi.unstubAllGlobals(); - _resetModelsDevForTests(); -}); - -function stubFetch(impl: () => Promise): ReturnType { - const mock = vi.fn().mockImplementation(impl); - vi.stubGlobal('fetch', mock); - return mock; -} - -describe('getModelsDevIndex', () => { - it('parses providers and model limits from api.json', async () => { - stubFetch(async () => new Response(JSON.stringify(API_JSON), { status: 200 })); - const index = await getModelsDevIndex(); - const sonnet = lookupModelsDev(index, 'anthropic', 'claude-sonnet-4-6'); - expect(sonnet?.limit).toEqual({ context: 200_000, input: undefined, output: 64_000 }); - expect(sonnet?.reasoning).toBe(true); - }); - - it('returns an empty index on non-200', async () => { - stubFetch(async () => new Response('', { status: 503 })); - const index = await getModelsDevIndex(); - expect(index.size).toBe(0); - }); - - it('returns an empty index when fetch throws', async () => { - stubFetch(async () => { - throw new Error('network down'); - }); - const index = await getModelsDevIndex(); - expect(index.size).toBe(0); - }); - - it('returns an empty index on malformed JSON', async () => { - stubFetch(async () => new Response('not-json', { status: 200 })); - const index = await getModelsDevIndex(); - expect(index.size).toBe(0); - }); - - it('caches the result: two calls, one fetch', async () => { - const mock = stubFetch(async () => new Response(JSON.stringify(API_JSON), { status: 200 })); - await getModelsDevIndex(); - await getModelsDevIndex(); - expect(mock).toHaveBeenCalledTimes(1); - expect(mock).toHaveBeenCalledWith(MODELSDEV_URL, expect.anything()); - }); - - it('dedupes concurrent fetches in flight', async () => { - const mock = stubFetch(async () => new Response(JSON.stringify(API_JSON), { status: 200 })); - await Promise.all([getModelsDevIndex(), getModelsDevIndex(), getModelsDevIndex()]); - expect(mock).toHaveBeenCalledTimes(1); - }); - - it('caches a failed fetch: does not re-hit models.dev within the failure TTL', async () => { - const mock = stubFetch(async () => new Response('', { status: 503 })); - const first = await getModelsDevIndex(); - const second = await getModelsDevIndex(); - expect(first.size).toBe(0); - expect(second.size).toBe(0); - expect(mock).toHaveBeenCalledTimes(1); - }); - - it('rejects out-of-range limit values (sanity bounds)', async () => { - const poisoned = { - anthropic: { - id: 'anthropic', - models: { - 'claude-x': { id: 'claude-x', limit: { context: 200_000, output: 1 } }, - 'claude-y': { id: 'claude-y', limit: { context: 200_000, output: 99_000_000 } }, - }, - }, - }; - stubFetch(async () => new Response(JSON.stringify(poisoned), { status: 200 })); - const idx = await getModelsDevIndex(); - expect(lookupModelsDev(idx, 'anthropic', 'claude-x')?.limit?.output).toBeUndefined(); - expect(lookupModelsDev(idx, 'anthropic', 'claude-y')?.limit?.output).toBeUndefined(); - // context within range still accepted - expect(lookupModelsDev(idx, 'anthropic', 'claude-x')?.limit?.context).toBe(200_000); - }); -}); - -describe('lookupModelsDev', () => { - async function index() { - stubFetch(async () => new Response(JSON.stringify(API_JSON), { status: 200 })); - return getModelsDevIndex(); - } - - it('matches exact model ids', async () => { - const idx = await index(); - expect(lookupModelsDev(idx, 'openai', 'gpt-5-pro')?.limit?.output).toBe(272_000); - }); - - it('matches date-suffixed ids against the undated catalog id', async () => { - const idx = await index(); - expect(lookupModelsDev(idx, 'anthropic', 'claude-sonnet-4-20250514')?.id).toBe( - 'claude-sonnet-4', - ); - }); - - it('maps the kimi provider id to moonshotai', async () => { - const idx = await index(); - expect(lookupModelsDev(idx, 'kimi', 'kimi-k2.6')?.limit?.context).toBe(256_000); - }); - - it.each(['lmstudio', 'llamacpp', 'unknown'])('returns undefined for provider %s', async (p) => { - const idx = await index(); - expect(lookupModelsDev(idx, p, 'anything')).toBeUndefined(); - }); - - it('returns undefined for unknown model ids', async () => { - const idx = await index(); - expect(lookupModelsDev(idx, 'anthropic', 'claude-nonexistent-9')).toBeUndefined(); - }); - - it('matches an undated lookup id against a date-suffixed catalog id (loop scan)', async () => { - const json = { - anthropic: { - id: 'anthropic', - models: { - 'claude-opus-4-20251101': { - id: 'claude-opus-4-20251101', - limit: { context: 200_000, output: 32_000 }, - }, - }, - }, - }; - stubFetch(async () => new Response(JSON.stringify(json), { status: 200 })); - const idx = await getModelsDevIndex(); - expect(lookupModelsDev(idx, 'anthropic', 'claude-opus-4')?.id).toBe('claude-opus-4-20251101'); - }); -}); diff --git a/harness/tests/runtime/openai-compat-url.test.ts b/harness/tests/runtime/openai-compat-url.test.ts deleted file mode 100644 index e75dac0fa..000000000 --- a/harness/tests/runtime/openai-compat-url.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { normalizeChatCompletionsUrl } from '../../src/runtime/openai-compat-url.js'; - -describe('normalizeChatCompletionsUrl', () => { - // The regression this test exists for: a user who saved - // `http://192.168.1.206:8080` from the Providers UI got 404 on every - // chat request because the override URL was POSTed verbatim. The fix - // routes the override through this helper. - it('appends /v1/chat/completions to a bare base URL', () => { - expect(normalizeChatCompletionsUrl('http://192.168.1.206:8080')).toBe( - 'http://192.168.1.206:8080/v1/chat/completions', - ); - }); - - it('appends /v1/chat/completions to a base URL with a trailing slash', () => { - expect(normalizeChatCompletionsUrl('http://localhost:8080/')).toBe( - 'http://localhost:8080/v1/chat/completions', - ); - }); - - // Regression: substring matching on the raw input previously concatenated - // the path into the query string, producing `?foo=bar/v1/chat/completions` - // and 404s on every request. Routing through the parsed URL fixes the - // path placement and preserves the query. - it('places /v1/chat/completions on the path even when the input has only a query string', () => { - expect(normalizeChatCompletionsUrl('http://localhost:8080?foo=bar')).toBe( - 'http://localhost:8080/v1/chat/completions?foo=bar', - ); - }); - - it('leaves a fully-qualified URL unchanged', () => { - expect(normalizeChatCompletionsUrl('http://localhost:8080/v1/chat/completions')).toBe( - 'http://localhost:8080/v1/chat/completions', - ); - }); - - it('treats any path containing /chat/completions as already qualified', () => { - // Some forks of llama-server / LM Studio expose the endpoint at a - // non-/v1 path. Don't double-append. - expect(normalizeChatCompletionsUrl('http://host:8080/api/v2/chat/completions')).toBe( - 'http://host:8080/api/v2/chat/completions', - ); - }); - - it('accepts https URLs', () => { - expect(normalizeChatCompletionsUrl('https://tunnel.ngrok.io')).toBe( - 'https://tunnel.ngrok.io/v1/chat/completions', - ); - }); - - it('returns null for empty / whitespace strings', () => { - expect(normalizeChatCompletionsUrl('')).toBeNull(); - expect(normalizeChatCompletionsUrl(' ')).toBeNull(); - }); - - it('returns null for non-http(s) schemes', () => { - expect(normalizeChatCompletionsUrl('file:///etc/passwd')).toBeNull(); - expect(normalizeChatCompletionsUrl('javascript:alert(1)')).toBeNull(); - }); - - it('returns null for unparseable URLs', () => { - expect(normalizeChatCompletionsUrl('not a url')).toBeNull(); - expect(normalizeChatCompletionsUrl('http://')).toBeNull(); - }); -}); diff --git a/harness/tests/runtime/output-tokens.test.ts b/harness/tests/runtime/output-tokens.test.ts deleted file mode 100644 index b281eab63..000000000 --- a/harness/tests/runtime/output-tokens.test.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { - _resetOutputTokenCapForTests, - clampOutputTokens, - getCatalogModel, - OUTPUT_TOKEN_MAX, - OUTPUT_TOKEN_MAX_ENV, - outputTokenCap, -} from '../../src/runtime/output-tokens.js'; -import type { ISdk } from '../../src/runtime/iii.js'; - -afterEach(() => { - delete process.env[OUTPUT_TOKEN_MAX_ENV]; - _resetOutputTokenCapForTests(); -}); - -describe('outputTokenCap', () => { - it('defaults to 32_000', () => { - expect(outputTokenCap()).toBe(OUTPUT_TOKEN_MAX); - }); - - it('reads a valid env override', () => { - process.env[OUTPUT_TOKEN_MAX_ENV] = '16000'; - expect(outputTokenCap()).toBe(16_000); - }); - - it.each(['abc', '0', '-5', 'NaN'])('falls back to 32_000 on invalid env %s', (v) => { - process.env[OUTPUT_TOKEN_MAX_ENV] = v; - expect(outputTokenCap()).toBe(OUTPUT_TOKEN_MAX); - }); -}); - -describe('clampOutputTokens precedence', () => { - const workerDefault = 8_192; - - it('defaults to min(modelMax, cap): model below cap', () => { - expect(clampOutputTokens({ modelMaxOutput: 16_000, userOverride: null, workerDefault })).toBe( - 16_000, - ); - }); - - it('defaults to min(modelMax, cap): model above cap clamps to 32k', () => { - expect(clampOutputTokens({ modelMaxOutput: 64_000, userOverride: null, workerDefault })).toBe( - 32_000, - ); - }); - - it('user override wins below the model ceiling', () => { - expect(clampOutputTokens({ modelMaxOutput: 64_000, userOverride: 50_000, workerDefault })).toBe( - 50_000, - ); - }); - - it('user override is clamped down to the model ceiling', () => { - expect( - clampOutputTokens({ modelMaxOutput: 64_000, userOverride: 100_000, workerDefault }), - ).toBe(64_000); - }); - - it('user override is NOT capped to 32k (deliberate choice)', () => { - expect( - clampOutputTokens({ modelMaxOutput: 128_000, userOverride: 64_000, workerDefault }), - ).toBe(64_000); - }); - - it('user override passes through when model is unknown', () => { - expect( - clampOutputTokens({ modelMaxOutput: undefined, userOverride: 4_096, workerDefault }), - ).toBe(4_096); - }); - - it.each([0, undefined, null])('unknown model max (%s) falls back to workerDefault', (v) => { - expect(clampOutputTokens({ modelMaxOutput: v, userOverride: null, workerDefault })).toBe( - workerDefault, - ); - }); - - it('honors an explicit cap argument over the env cap', () => { - expect( - clampOutputTokens({ modelMaxOutput: 64_000, userOverride: null, workerDefault, cap: 16_000 }), - ).toBe(16_000); - }); - - it('env cap raises the default for high-output models', () => { - process.env[OUTPUT_TOKEN_MAX_ENV] = '64000'; - expect(clampOutputTokens({ modelMaxOutput: 128_000, userOverride: null, workerDefault })).toBe( - 64_000, - ); - }); - - it('floors fractional values so token counts stay integers', () => { - expect(clampOutputTokens({ modelMaxOutput: 64_000, userOverride: 1024.5, workerDefault })).toBe( - 1024, - ); - expect(clampOutputTokens({ modelMaxOutput: undefined, userOverride: 1.5, workerDefault })).toBe( - 1, - ); - }); - - it.each([0, -1])('ignores a non-positive userOverride (%s)', (ov) => { - expect(clampOutputTokens({ modelMaxOutput: 64_000, userOverride: ov, workerDefault })).toBe( - 32_000, - ); - expect(clampOutputTokens({ modelMaxOutput: undefined, userOverride: ov, workerDefault })).toBe( - workerDefault, - ); - }); - - it('rejects scientific-notation env values instead of truncating them', () => { - // parseInt("1e9") === 1 — the strict digits-only parse must fall back to 32k. - process.env[OUTPUT_TOKEN_MAX_ENV] = '1e9'; - expect(outputTokenCap()).toBe(OUTPUT_TOKEN_MAX); - }); -}); - -describe('getCatalogModel', () => { - function fakeIii(trigger: (req: unknown) => Promise): ISdk { - return { trigger } as unknown as ISdk; - } - - it('returns the catalog entry from models::get', async () => { - const entry = { id: 'claude-sonnet-4-6', provider: 'anthropic', max_output_tokens: 64_000 }; - const trigger = vi.fn().mockResolvedValue(entry); - const out = await getCatalogModel(fakeIii(trigger), 'anthropic', 'claude-sonnet-4-6'); - expect(out).toEqual(entry); - expect(trigger).toHaveBeenCalledWith( - expect.objectContaining({ - function_id: 'models::get', - payload: { provider: 'anthropic', model_id: 'claude-sonnet-4-6' }, - }), - ); - }); - - it('returns null when the model is unknown', async () => { - const out = await getCatalogModel(fakeIii(vi.fn().mockResolvedValue(null)), 'anthropic', 'x'); - expect(out).toBeNull(); - }); - - it('returns null when the bus call throws', async () => { - const out = await getCatalogModel( - fakeIii(vi.fn().mockRejectedValue(new Error('timeout'))), - 'anthropic', - 'x', - ); - expect(out).toBeNull(); - }); -}); diff --git a/harness/tests/turn-orchestrator/context-view.test.ts b/harness/tests/turn-orchestrator/context-view.test.ts index 2322f0e0e..7175f95d8 100644 --- a/harness/tests/turn-orchestrator/context-view.test.ts +++ b/harness/tests/turn-orchestrator/context-view.test.ts @@ -26,6 +26,12 @@ function entry(id: string, message: AgentMessage): MessageWithEntryId { return { entry_id: id, message }; } +/** buildSummaryMessage stamps Date.now(); the view under test calls it again, + * so an exact-timestamp expectation races the millisecond boundary. */ +function summary(text: string) { + return { ...buildSummaryMessage(text), timestamp: expect.any(Number) }; +} + describe('buildContextView', () => { it('returns raw path when there is no compaction', () => { const messages = [entry('a', user('one')), entry('b', asst('two'))]; @@ -47,7 +53,7 @@ describe('buildContextView', () => { const compactions = [{ summary: 'condensed', tail_start_id: 'tail1', timestamp: 100 }]; expect(buildContextView(messages, compactions)).toEqual([ - buildSummaryMessage('condensed'), + summary('condensed'), asst('keep'), user('in flight'), ]); @@ -64,10 +70,7 @@ describe('buildContextView', () => { { summary: 'latest', tail_start_id: 't2', timestamp: 20 }, ]; - expect(buildContextView(messages, compactions)).toEqual([ - buildSummaryMessage('latest'), - user('recent'), - ]); + expect(buildContextView(messages, compactions)).toEqual([summary('latest'), user('recent')]); }); it('keeps the whole tail when tail_start_id is absent from the path', () => { @@ -75,7 +78,7 @@ describe('buildContextView', () => { const compactions = [{ summary: 's', tail_start_id: 'gone', timestamp: 1 }]; expect(buildContextView(messages, compactions)).toEqual([ - buildSummaryMessage('s'), + summary('s'), user('one'), asst('two'), ]); diff --git a/harness/tests/turn-orchestrator/preflight.test.ts b/harness/tests/turn-orchestrator/preflight.test.ts index 1d265248f..2a09833b9 100644 --- a/harness/tests/turn-orchestrator/preflight.test.ts +++ b/harness/tests/turn-orchestrator/preflight.test.ts @@ -24,8 +24,9 @@ function makeIii(overrides?: { trigger: async <_T, R>(req: { function_id: string; payload: unknown }): Promise => { calls.push({ function_id: req.function_id, payload: req.payload }); - if (req.function_id === 'models::get') { - return (overrides?.modelsGetResult ?? null) as R; + if (req.function_id === 'router::models::get') { + const m = overrides?.modelsGetResult ?? null; + return (m ? { model: m } : null) as R; } if (req.function_id === 'session::messages') { return (overrides?.sessionTreeResult ?? { messages: [] }) as R; @@ -54,7 +55,7 @@ describe('runPreflight', () => { expect(calls.some((c) => c.function_id === 'context-compaction::compact_now')).toBe(false); }); - it('returns ok and skips compact_now when models::get returns null', async () => { + it('returns ok and skips compact_now when router::models::get returns null', async () => { const { iii, calls } = makeIii({ modelsGetResult: null }); const result = await runPreflight(iii, 'session-1', [smallMessage], 'anthropic', 'claude-3'); @@ -119,7 +120,7 @@ describe('runPreflight', () => { expect(result).toBe('ok'); }); - it('uses the pre-resolved model and skips models::get', async () => { + it('uses the pre-resolved model and skips router::models::get', async () => { const { iii, calls } = makeIii({ modelsGetResult: { context_window: 1, max_output_tokens: 0 }, }); @@ -142,11 +143,11 @@ describe('runPreflight', () => { ); expect(result).toBe('ok'); - expect(calls.some((c) => c.function_id === 'models::get')).toBe(false); + expect(calls.some((c) => c.function_id === 'router::models::get')).toBe(false); expect(calls.some((c) => c.function_id === 'context-compaction::compact_now')).toBe(false); }); - it('fetches models::get when no pre-resolved model is threaded', async () => { + it('fetches router::models::get when no pre-resolved model is threaded', async () => { const { iii, calls } = makeIii({ modelsGetResult: { context_window: 200_000, max_output_tokens: 8_096 }, }); @@ -154,7 +155,7 @@ describe('runPreflight', () => { const result = await runPreflight(iii, 'session-1', [smallMessage], 'anthropic', 'claude-3'); expect(result).toBe('ok'); - expect(calls.some((c) => c.function_id === 'models::get')).toBe(true); + expect(calls.some((c) => c.function_id === 'router::models::get')).toBe(true); }); it('passes session_id and model info to compact_now', async () => { diff --git a/harness/tests/turn-orchestrator/provider-router.test.ts b/harness/tests/turn-orchestrator/provider-router.test.ts deleted file mode 100644 index 0f27bb07b..000000000 --- a/harness/tests/turn-orchestrator/provider-router.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - buildInput, - decide, - targetFunctionId, -} from '../../src/turn-orchestrator/provider-router.js'; - -describe('decide', () => { - it('routes anthropic when provider=anthropic', () => { - expect(decide({ provider: 'anthropic', model: 'claude' }).provider).toBe('anthropic'); - }); - - it('routes openai when provider=openai', () => { - expect(decide({ provider: 'openai', model: 'gpt-5' }).provider).toBe('openai'); - }); - - it('routes kimi when provider=kimi', () => { - expect(decide({ provider: 'kimi', model: 'kimi-k2-0905-preview' }).provider).toBe('kimi'); - }); - - it('routes llamacpp when provider=llamacpp', () => { - expect(decide({ provider: 'llamacpp', model: 'Meta-Llama-3.1-8B' }).provider).toBe('llamacpp'); - expect(decide({ model: 'Meta-Llama-3.1-8B' }).provider).toBe('anthropic'); - }); - - it('maps targetFunctionId for llamacpp', () => { - expect(targetFunctionId({ provider: 'llamacpp', model: 'm' })).toBe( - 'provider::llamacpp::stream', - ); - }); - - it('routes lmstudio when provider=lmstudio (no model-name heuristic)', () => { - expect(decide({ provider: 'lmstudio', model: 'qwen/qwen3-4b-2507' }).provider).toBe('lmstudio'); - expect( - decide({ - provider: 'lmstudio', - model: 'lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF', - }).provider, - ).toBe('lmstudio'); - const ambiguousIds = [ - 'qwen/qwen3-4b-2507', - 'google/gemma-2-9b-it', - 'google/gemma-3-e4b', - 'meta-llama/Llama-3-70B', - 'mistralai/Mistral-7B-v0.3', - 'TheBloke/Llama-2-7B-GGUF', - 'lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF', - 'deepseek-ai/DeepSeek-R1', - ]; - for (const model of ambiguousIds) { - expect(decide({ model }).provider, `model=${model}`).toBe('anthropic'); - } - }); - - it('falls back to model heuristic when provider missing', () => { - expect(decide({ model: 'gpt-5' }).provider).toBe('openai'); - expect(decide({ model: 'claude-opus-4-7' }).provider).toBe('anthropic'); - expect(decide({ model: 'kimi-k2-0905-preview' }).provider).toBe('kimi'); - expect(decide({ model: 'kimi-k2-turbo-preview' }).provider).toBe('kimi'); - expect(decide({ model: 'kimi-k2.6' }).provider).toBe('kimi'); - expect(decide({ model: 'moonshot-v1-128k' }).provider).toBe('kimi'); - expect(decide({ model: 'moonshot-v1-8k-vision-preview' }).provider).toBe('kimi'); - }); -}); - -describe('targetFunctionId', () => { - it('maps decisions to provider stream function ids', () => { - expect(targetFunctionId({ provider: 'anthropic', model: 'm' })).toBe( - 'provider::anthropic::stream', - ); - expect(targetFunctionId({ provider: 'openai', model: 'm' })).toBe('provider::openai::stream'); - expect(targetFunctionId({ provider: 'kimi', model: 'm' })).toBe('provider::kimi::stream'); - expect(targetFunctionId({ provider: 'lmstudio', model: 'm' })).toBe( - 'provider::lmstudio::stream', - ); - }); -}); - -describe('buildInput', () => { - it('roundtrips the canonical fields', () => { - const input = buildInput( - { provider: 'anthropic', model: 'claude' }, - { channel_id: 'c', access_key: 'k', direction: 'write' }, - 'sys', - [{ role: 'user', content: [{ type: 'text', text: 'hi' }], timestamp: 0 }], - [{ name: 'agent_trigger', description: 'd', parameters: {} }], - ); - expect(input.model).toBe('claude'); - expect(input.tools).toHaveLength(1); - expect(input.writer_ref.channel_id).toBe('c'); - }); - - it('carries thinking_level when provided and omits it when absent', () => { - const decision = { provider: 'anthropic', model: 'claude' } as const; - const writer = { channel_id: 'c', access_key: 'k', direction: 'write' } as const; - const withLevel = buildInput(decision, writer, 'sys', [], [], 'high'); - expect(withLevel.thinking_level).toBe('high'); - const withoutLevel = buildInput(decision, writer, 'sys', [], []); - expect(withoutLevel).not.toHaveProperty('thinking_level'); - }); - - it('carries model_meta when provided and omits it when absent', () => { - const decision = { provider: 'anthropic', model: 'claude-sonnet-4-6' } as const; - const writer = { channel_id: 'c', access_key: 'k', direction: 'write' } as const; - const model_meta = { - id: 'claude-sonnet-4-6', - provider: 'anthropic', - api: 'anthropic-messages', - display_name: 'Claude Sonnet 4.6', - context_window: 1_000_000, - max_output_tokens: 64_000, - }; - const withMeta = buildInput(decision, writer, 'sys', [], [], undefined, model_meta); - expect(withMeta.model_meta).toEqual(model_meta); - const withoutMeta = buildInput(decision, writer, 'sys', [], []); - expect(withoutMeta).not.toHaveProperty('model_meta'); - }); - - it('carries resolution_key when provided and omits it when absent', () => { - const decision = { provider: 'anthropic', model: 'claude' } as const; - const writer = { channel_id: 'c', access_key: 'k', direction: 'write' } as const; - const withKey = buildInput( - decision, - writer, - 'sys', - [], - [], - undefined, - undefined, - 1738000000000, - ); - expect(withKey.resolution_key).toBe(1738000000000); - const withoutKey = buildInput(decision, writer, 'sys', [], []); - expect(withoutKey).not.toHaveProperty('resolution_key'); - }); -}); diff --git a/harness/tests/turn-orchestrator/provisioning-layer.test.ts b/harness/tests/turn-orchestrator/provisioning-layer.test.ts index 1e6ba47e4..3da6cb5fc 100644 --- a/harness/tests/turn-orchestrator/provisioning-layer.test.ts +++ b/harness/tests/turn-orchestrator/provisioning-layer.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { Model } from '../../src/models-catalog/types.js'; +import type { Model } from '../../src/types/model.js'; import type { ProvisioningPorts } from '../../src/turn-orchestrator/provisioning/ports.js'; import { applyProvisioningOutcome, @@ -26,6 +26,7 @@ function stubPorts(overrides: Partial = {}): ProvisioningPort function_schemas: [], })), saveRunRequest: vi.fn(async () => {}), + route: vi.fn(async (provider: string, _model: string) => provider || 'anthropic'), resolveModel: vi.fn(async () => null), ...overrides, }; @@ -71,7 +72,8 @@ describe('processProvisioning', () => { expect(outcome.runRequest.system_prompt).toBe('custom override'); }); - it('resolves the model once against the DECIDED provider and returns it', async () => { + it('resolves the model once against the ROUTED provider and pins it on the request', async () => { + const route = vi.fn(async () => 'openai'); const resolveModel = vi.fn(async () => FAKE_MODEL); const ports = stubPorts({ loadRunRequest: vi.fn(async () => ({ @@ -81,17 +83,44 @@ describe('processProvisioning', () => { system_prompt: '', function_schemas: [], })), + route, resolveModel, }); const rec = { ...newRecord('s1'), state: 'provisioning' as const }; const outcome = await processProvisioning(ports, rec); + expect(route).toHaveBeenCalledTimes(1); + expect(route).toHaveBeenCalledWith('', 'gpt-4'); expect(resolveModel).toHaveBeenCalledTimes(1); expect(resolveModel).toHaveBeenCalledWith('openai', 'gpt-4'); + expect(outcome.runRequest.routed_provider).toBe('openai'); expect(outcome.model_meta).toEqual(FAKE_MODEL); }); + it('skips model resolution and serves the default-family prompt when routing fails', async () => { + const route = vi.fn(async () => null); + const resolveModel = vi.fn(async () => FAKE_MODEL); + const ports = stubPorts({ + loadRunRequest: vi.fn(async () => ({ + provider: '', + model: 'gpt-4', + mode: null, + system_prompt: '', + function_schemas: [], + })), + route, + resolveModel, + }); + const rec = { ...newRecord('s1'), state: 'provisioning' as const }; + + const outcome = await processProvisioning(ports, rec); + + expect(resolveModel).not.toHaveBeenCalled(); + expect(outcome.runRequest.routed_provider).toBe(''); + expect(outcome.model_meta).toBeNull(); + }); + it('does not resolve when there is no model id', async () => { const resolveModel = vi.fn(async () => FAKE_MODEL); const ports = stubPorts({ resolveModel }); diff --git a/harness/tests/turn-orchestrator/run-start.test.ts b/harness/tests/turn-orchestrator/run-start.test.ts index dbf3b0bc4..01c8fdbc9 100644 --- a/harness/tests/turn-orchestrator/run-start.test.ts +++ b/harness/tests/turn-orchestrator/run-start.test.ts @@ -110,6 +110,19 @@ describe('RunStartPayloadSchema', () => { expect(() => RunStartPayloadSchema.parse(null)).toThrow(); expect(() => RunStartPayloadSchema.parse(undefined)).toThrow(); }); + + it('accepts a thinking_level and rejects unknown levels', () => { + // Threaded to router::chat via the persisted run request; an invalid + // level must fail at the boundary, not surface as a provider warning. + expect( + RunStartPayloadSchema.parse({ ...harnessRunStartPayload, thinking_level: 'high' }) + .thinking_level, + ).toBe('high'); + expect(RunStartPayloadSchema.parse(harnessRunStartPayload).thinking_level).toBeUndefined(); + expect(() => + RunStartPayloadSchema.parse({ ...harnessRunStartPayload, thinking_level: 'ultra' }), + ).toThrow(); + }); }); describe('register', () => { diff --git a/harness/tests/turn-orchestrator/system-prompt.test.ts b/harness/tests/turn-orchestrator/system-prompt.test.ts index bd38c0ace..36bdb9aa0 100644 --- a/harness/tests/turn-orchestrator/system-prompt.test.ts +++ b/harness/tests/turn-orchestrator/system-prompt.test.ts @@ -592,44 +592,40 @@ describe.each(VARIANTS)('invariant contract — %s variant', (_family, out) => { }); describe('promptFamily', () => { - it('routes explicit providers to their families', () => { - expect(promptFamily('anthropic', 'claude-opus-4-7')).toBe('anthropic'); - expect(promptFamily('openai', 'gpt-5')).toBe('gpt'); - expect(promptFamily('kimi', 'kimi-k2-0905-preview')).toBe('kimi'); - expect(promptFamily('lmstudio', 'qwen/qwen3-4b-2507')).toBe('default'); - expect(promptFamily('llamacpp', 'Meta-Llama-3.1-8B')).toBe('default'); + it('maps the ROUTED provider to its family — routing itself lives in the llm-router', () => { + expect(promptFamily('anthropic')).toBe('anthropic'); + expect(promptFamily('openai')).toBe('gpt'); + expect(promptFamily('kimi')).toBe('kimi'); + expect(promptFamily('lmstudio')).toBe('default'); + expect(promptFamily('llamacpp')).toBe('default'); }); - it('falls back to model heuristics when provider is empty', () => { - expect(promptFamily('', 'gpt-4')).toBe('gpt'); - expect(promptFamily('', 'o3-mini')).toBe('gpt'); - expect(promptFamily('', 'kimi-k2-0905-preview')).toBe('kimi'); - expect(promptFamily('', 'moonshot-v1-128k')).toBe('kimi'); + it('serves the anthropic family when no provider routed (router unreachable)', () => { + // Mirrors the llm-router entry's seeded default_provider, so the + // un-routed prompt matches what the routed turn would have served. + expect(promptFamily('')).toBe('anthropic'); }); - it('defaults to anthropic when nothing matches', () => { - expect(promptFamily('', '')).toBe('anthropic'); - // Local model ids require an explicit provider (the router pins this); a - // bare HF-style id without provider stays on the anthropic route. - expect(promptFamily('', 'qwen-7b')).toBe('anthropic'); + it('serves the generic default for an unrecognized provider id', () => { + expect(promptFamily('some-new-provider')).toBe('default'); }); }); describe('buildSystemPrompt variant selection', () => { it('serves the gpt variant (persistence voice) for openai runs', () => { - const out = buildSystemPrompt({ provider: 'openai', model: 'gpt-5' }); + const out = buildSystemPrompt({ provider: 'openai' }); expect(out).toContain('## Autonomy and persistence'); expect(out).toMatch(/Persist until the task is fully handled\s+end-to-end/); }); it('serves the kimi variant (MUST imperatives) for kimi runs', () => { - const out = buildSystemPrompt({ provider: 'kimi', model: 'kimi-k2-0905-preview' }); + const out = buildSystemPrompt({ provider: 'kimi' }); expect(out).toContain('# Ultimate Reminders'); expect(out).toContain('# Prompt and Tool Use'); }); it('serves the default variant (step-by-step) for local runtimes', () => { - const out = buildSystemPrompt({ provider: 'lmstudio', model: 'qwen/qwen3-4b-2507' }); + const out = buildSystemPrompt({ provider: 'lmstudio' }); expect(out).toContain('Follow these steps for EVERY action'); expect(out).toContain('# Final checklist'); }); @@ -640,10 +636,10 @@ describe('buildSystemPrompt variant selection', () => { it('prepends the mode paragraph before the identity line on every variant', () => { const runs = [ - { provider: 'anthropic', model: 'claude-sonnet-4-6' }, - { provider: 'openai', model: 'gpt-5' }, - { provider: 'kimi', model: 'kimi-k2-0905-preview' }, - { provider: 'llamacpp', model: 'Meta-Llama-3.1-8B' }, + { provider: 'anthropic' }, + { provider: 'openai' }, + { provider: 'kimi' }, + { provider: 'llamacpp' }, ]; for (const run of runs) { const out = buildSystemPrompt({ ...run, mode: 'agent' }); @@ -658,7 +654,6 @@ describe('buildSystemPrompt variant selection', () => { override: 'custom-override', mode: 'plan', provider: 'openai', - model: 'gpt-5', }); expect(out).toBe('custom-override'); }); diff --git a/harness/tests/types/provider.test.ts b/harness/tests/types/provider.test.ts deleted file mode 100644 index 249072038..000000000 --- a/harness/tests/types/provider.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - ProviderStreamInputJsonSchema, - ProviderStreamInputSchema, - ProviderStreamOutputSchema, -} from '../../src/types/provider.js'; - -describe('ProviderStreamInputSchema', () => { - it('accepts minimum required fields', () => { - const ok = ProviderStreamInputSchema.parse({ - writer_ref: { channel_id: 'c', access_key: 'k', direction: 'write' }, - model: 'claude-3-5-sonnet', - messages: [], - }); - expect(ok.tools).toEqual([]); - }); - - it('rejects bad direction', () => { - expect(() => - ProviderStreamInputSchema.parse({ - writer_ref: { channel_id: 'c', access_key: 'k', direction: 'bidir' }, - model: 'm', - messages: [], - }), - ).toThrow(); - }); - - it('exposes a JSON schema with the writer_ref and model fields', () => { - const schema = ProviderStreamInputJsonSchema as Record; - expect(JSON.stringify(schema)).toContain('writer_ref'); - expect(JSON.stringify(schema)).toContain('model'); - }); - - it('passes a sparse model_meta through instead of failing the stream', () => { - const sparse = ProviderStreamInputSchema.parse({ - writer_ref: { channel_id: 'c', access_key: 'k', direction: 'write' }, - model: 'm', - messages: [], - model_meta: { id: 'm', provider: 'anthropic' }, - }); - expect(sparse.model_meta).toEqual({ id: 'm', provider: 'anthropic' }); - }); - - it('coerces a non-object model_meta to absent', () => { - const parsed = ProviderStreamInputSchema.parse({ - writer_ref: { channel_id: 'c', access_key: 'k', direction: 'write' }, - model: 'm', - messages: [], - model_meta: 'garbage', - }); - expect(parsed.model_meta).toBeUndefined(); - }); -}); - -describe('ProviderStreamOutputSchema', () => { - it('parses ok-only payloads', () => { - expect(ProviderStreamOutputSchema.parse({ ok: true }).ok).toBe(true); - }); -}); diff --git a/iii-permissions.yaml b/iii-permissions.yaml index db56ad874..79fc1abdb 100644 --- a/iii-permissions.yaml +++ b/iii-permissions.yaml @@ -19,12 +19,22 @@ rules: - '!state::delete' - '!stream::set' - '!iii::durable::publish' - # Provider credentials live in the `harness` configuration entry. Agents - # must never resolve a secret, self-register a provider, or read/rewrite - # the configuration value (which carries plaintext api keys). The console - # edits these as a user-initiated SDK call, which bypasses this gate. - - '!harness::provider::resolve' - - '!harness::provider::register' + # Provider credentials live in the `llm-router` configuration entry. Agents + # must never resolve a secret, self-register a provider, poison the model + # catalog, or read/rewrite the configuration value (which carries plaintext + # api keys). Direct router spend (chat/complete) bypasses the harness + # loop's accounting; the routing preview is orchestrator-internal. The + # console edits config as a user-initiated SDK call, which bypasses this gate. + - '!router::provider::resolve' + - '!router::provider::register' + - '!router::provider::update_credential' + - '!router::models::reconcile' + - '!router::chat' + - '!router::complete' + - '!router::abort' + - '!router::route' + - '!router::on_worker_available' + - '!router::on_config_changed' - '!configuration::get' - '!configuration::set' - '!configuration::register' @@ -38,7 +48,6 @@ rules: - '!oauth::anthropic::login' - '!oauth::openai-codex::login' - '!run::start' - - '!router::stream_assistant' # session-manager: deny-by-default for in-run agents (integration.md §2). # An agent that can write here can rewrite its own transcript; the raw # store protocol (session::store::*) bypasses every invariant. Reads stay @@ -65,9 +74,10 @@ rules: # Read-only / introspection (extend below for your tools). - state::get - state::list - - models::list - - models::get - - models::supports + - router::models::list + - router::models::get + - router::models::supports + - router::provider::list - oauth::anthropic::status - oauth::openai-codex::status - engine::functions::list diff --git a/llm-router/README.md b/llm-router/README.md index ef6e3470c..5e0c58704 100644 --- a/llm-router/README.md +++ b/llm-router/README.md @@ -53,6 +53,7 @@ partial content, so consumers never hang on a half-open stream. | `router::chat` | Stream a turn into the caller's channel; returns the turn summary. | | `router::complete` | Non-streaming convenience over the same pipeline; returns the final message. | | `router::abort` | Cancel an in-flight turn by `request_id`. | +| `router::route` | Read-only routing preview: `{model, provider?}` → `{provider, candidates}`, same rules and error codes as `router::chat`. Pin the result as the explicit `provider` on the chat call when you need the provider before streaming. | | `router::models::list` | List catalog models, filterable by `provider` / `capability`. | | `router::models::get` | Fetch one model record (`null` when unknown). | | `router::models::supports` | Check one capability flag for one model. | @@ -111,6 +112,19 @@ router diffs the changed slice, debounces ~2 s, and kicks that provider's catalog via `router::models::reconcile` and show up in `router::models::list` within seconds — no restart. +### Operational notes + +- **Env-var credential fallback resolves in the router's process.** A + provider's `credential_env_var` (e.g. `ANTHROPIC_API_KEY`) is read by the + llm-router binary, not by the provider worker — launch the router with + those variables set, or put keys in the entry. A key present only in + another worker's environment shows up as `configured: false`. +- **Registration-token recovery.** Re-registering a provider id without its + original token is rejected (anti-takeover). If a provider durably lost its + token, delete the router's registry state (iii-state scope `llm-router`, + key `registry`) and restart the affected providers to re-bind; pasted + credentials in the configuration entry are unaffected. + ## Events The router publishes three events over the engine's `iii-pubsub` worker. Bind diff --git a/llm-router/iii-permissions.yaml b/llm-router/iii-permissions.yaml index c747f8e96..ab0ba92c5 100644 --- a/llm-router/iii-permissions.yaml +++ b/llm-router/iii-permissions.yaml @@ -15,6 +15,8 @@ rules: - '!router::chat' - '!router::complete' - '!router::abort' + # Routing preview is orchestrator-internal; agents have no business probing it. + - '!router::route' # Safe read surface (enumerated — deliberately NOT router::models::*). - 'router::models::list' - 'router::models::get' diff --git a/llm-router/src/chat/chat.rs b/llm-router/src/chat/chat.rs index 55a04ce60..99922e913 100644 --- a/llm-router/src/chat/chat.rs +++ b/llm-router/src/chat/chat.rs @@ -181,6 +181,7 @@ impl ChatPipeline { max_output_tokens, &settings, &entry_handle, + &request_id, sink, ) .await; @@ -197,6 +198,7 @@ impl ChatPipeline { max_output_tokens: u64, settings: &RouterSettings, inflight: &super::inflight::InflightEntry, + request_id: &str, sink: Arc, ) -> Result { let pricing = model_meta.and_then(|m| m.pricing.clone()); @@ -227,18 +229,14 @@ impl ChatPipeline { let mut reader = channel.reader; inflight.set_closer(reader.closer()); - let stream_input = json!({ - "writer_ref": channel.writer_ref, - "system_prompt": call.system_prompt, - "model": call.model, - "messages": call.messages, - "tools": call.tools, - "response_format": call.response_format, - "thinking_level": call.thinking_level, - "max_output_tokens": max_output_tokens, - "provider_options": call.provider_options.as_ref().and_then(|o| o.get(provider)).cloned(), - "model_meta": model_meta, - }); + let stream_input = build_stream_input( + call, + provider, + serde_json::to_value(&channel.writer_ref).expect("serializable writer_ref"), + max_output_tokens, + model_meta, + request_id, + ); // The provider call runs concurrently with the relay; if it throws // pre-stream, closing the reader unblocks the loop immediately. @@ -445,6 +443,61 @@ fn rand_unit() -> f64 { (Uuid::new_v4().as_u128() % 1000) as f64 / 1000.0 } +/// Build the per-attempt `provider::::stream` payload (the wire shape of +/// `types::router::ProviderStreamInput`). Optional fields are omitted, never +/// null — provider-side schemas reject `null` where a string or array is +/// expected. `resolution_key` is the request id: stable across retry attempts +/// within a turn, fresh per turn, so providers can dedupe per-turn credential +/// resolution. +fn build_stream_input( + call: &ChatCall, + provider: &str, + writer_ref: Value, + max_output_tokens: u64, + model_meta: Option<&crate::types::model::Model>, + request_id: &str, +) -> Value { + let mut input = serde_json::Map::new(); + input.insert("writer_ref".into(), writer_ref); + input.insert("model".into(), Value::String(call.model.clone())); + input.insert("messages".into(), call.messages.clone()); + input.insert("max_output_tokens".into(), json!(max_output_tokens)); + input.insert( + "resolution_key".into(), + Value::String(request_id.to_string()), + ); + insert_present( + &mut input, + "system_prompt", + call.system_prompt.clone().map(Value::String), + ); + insert_present(&mut input, "tools", call.tools.clone()); + insert_present(&mut input, "response_format", call.response_format.clone()); + insert_present(&mut input, "thinking_level", call.thinking_level.clone()); + insert_present( + &mut input, + "provider_options", + call.provider_options + .as_ref() + .and_then(|o| o.get(provider)) + .cloned(), + ); + insert_present( + &mut input, + "model_meta", + model_meta.and_then(|m| serde_json::to_value(m).ok()), + ); + Value::Object(input) +} + +fn insert_present(map: &mut serde_json::Map, key: &str, value: Option) { + if let Some(v) = value { + if !v.is_null() { + map.insert(key.to_string(), v); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -530,4 +583,76 @@ mod tests { "exactly one frame expected, got a second: {second:?}" ); } + + /// Provider-side schemas reject `null` where a string or array is + /// expected, so absent options must be omitted keys, never null values. + /// `resolution_key` must always be present and stable across the retry + /// attempts of one request. + #[test] + fn stream_input_omits_absent_options_and_carries_resolution_key() { + let call: ChatCall = serde_json::from_value(json!({ + "model": "claude-test", + "messages": [], + })) + .unwrap(); + let writer_ref = json!({ "channel_id": "c", "access_key": "k", "direction": "write" }); + + let attempt1 = build_stream_input( + &call, + "anthropic", + writer_ref.clone(), + 32_000, + None, + "req-1", + ); + let attempt2 = build_stream_input( + &call, + "anthropic", + writer_ref.clone(), + 32_000, + None, + "req-1", + ); + + let obj = attempt1.as_object().unwrap(); + for absent in [ + "system_prompt", + "tools", + "response_format", + "thinking_level", + "provider_options", + "model_meta", + ] { + assert!( + !obj.contains_key(absent), + "{absent} must be omitted, not null" + ); + } + assert!( + obj.values().all(|v| !v.is_null()), + "no null values on the wire" + ); + assert_eq!(obj["resolution_key"], json!("req-1")); + assert_eq!(obj["max_output_tokens"], json!(32_000)); + assert_eq!( + attempt1["resolution_key"], attempt2["resolution_key"], + "resolution_key is stable across attempts of one request" + ); + + // Present options ride through, and provider_options narrows to this + // provider's slice. + let call: ChatCall = serde_json::from_value(json!({ + "model": "claude-test", + "messages": [], + "system_prompt": "be brief", + "tools": [{ "name": "t", "description": "d", "parameters": {} }], + "thinking_level": "high", + "provider_options": { "anthropic": { "beta": true }, "openai": { "x": 1 } }, + })) + .unwrap(); + let input = build_stream_input(&call, "anthropic", writer_ref, 8192, None, "req-2"); + assert_eq!(input["system_prompt"], json!("be brief")); + assert_eq!(input["thinking_level"], json!("high")); + assert_eq!(input["provider_options"], json!({ "beta": true })); + } } diff --git a/llm-router/src/register.rs b/llm-router/src/register.rs index b8df5575c..638c0dfc8 100644 --- a/llm-router/src/register.rs +++ b/llm-router/src/register.rs @@ -111,6 +111,14 @@ pub async fn register_router(iii: III) -> Result { "router::provider::list", RegisterFunction::new_async(make_provider_list(iii.clone(), registry.clone())), ); + iii.register_function( + "router::route", + RegisterFunction::new_async(crate::routing::make_route( + registry.clone(), + catalog.clone(), + settings.clone(), + )), + ); iii.register_function( "router::provider::register", RegisterFunction::new_async(make_provider_register( diff --git a/llm-router/src/routing.rs b/llm-router/src/routing.rs index 43c74046b..10215ac37 100644 --- a/llm-router/src/routing.rs +++ b/llm-router/src/routing.rs @@ -1,5 +1,18 @@ //! decide(): ordered candidate list (spec § Routing). MVP consumes //! candidates[0]; the list shape is the future fallback seam. +//! `router::route` exposes the same decision as a read-only preview so +//! consumers that need the provider before streaming (prompt selection, +//! provisioning metadata) can pin it as the explicit `provider` on +//! `router::chat` — preview and execution can never diverge. +use std::sync::{Arc, RwLock}; + +use futures::future::BoxFuture; +use iii_sdk::IIIError; +use serde_json::{json, Value}; + +use crate::catalog::store::CatalogStore; +use crate::registry::store::RegistryStore; +use crate::settings::RouterSettings; use crate::types::errors::{RouterCode, RouterError}; #[derive(Debug, Clone, PartialEq)] @@ -82,6 +95,49 @@ pub fn decide(input: &DecideInput) -> Result, RouterError> { )) } +/// The `router::route` iii function: `{model, provider?}` → +/// `{provider, candidates}`. Same inputs, same `decide()`, same error codes +/// as the chat pipeline's routing step — just without the stream. +pub fn make_route( + registry: Arc, + catalog: Arc, + settings: Arc>, +) -> impl Fn(Value) -> BoxFuture<'static, Result> + Send + Sync + 'static { + move |raw: Value| { + let (registry, catalog, settings) = (registry.clone(), catalog.clone(), settings.clone()); + Box::pin(async move { + let model = raw + .get("model") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + if model.is_empty() { + return Err( + RouterError::new(RouterCode::InvalidRequest, "model is required").into(), + ); + } + let provider = raw + .get("provider") + .and_then(Value::as_str) + .map(String::from); + let (heuristics, default_provider) = { + let s = settings.read().unwrap(); + (s.routing_heuristics.clone(), s.default_provider.clone()) + }; + let candidates = decide(&DecideInput { + model, + provider, + registered_providers: registry.ids().await, + catalog: catalog.model_ids().await, + heuristics, + default_provider, + }) + .map_err(IIIError::from)?; + Ok(json!({ "provider": candidates[0], "candidates": candidates })) + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/llm-router/tests/integration.rs b/llm-router/tests/integration.rs index 44a832199..0e261ec65 100644 --- a/llm-router/tests/integration.rs +++ b/llm-router/tests/integration.rs @@ -406,6 +406,58 @@ async fn end_to_end_relay_over_a_live_engine() { router_iii.shutdown(); } +/// `router::route` must preview exactly the provider `router::chat` would +/// execute on, and throw the same typed codes when nothing routes — consumers +/// pin the preview as the explicit `provider` on the chat call. +#[tokio::test(flavor = "multi_thread")] +async fn route_previews_the_same_provider_chat_executes() { + let engine = engine_or_skip!(); + + let router_iii = register_worker(&engine.url, InitOptions::default()); + register_router(router_iii.clone()) + .await + .expect("router boots"); + let _provider = start_live_provider(&engine.url, ProviderOptions::default()).await; + + let consumer = register_worker(&engine.url, InitOptions::default()); + + // catalog-owner routing: live-1 sits in provider "real"'s static slice. + let route = call(&consumer, "router::route", json!({ "model": "live-1" })) + .await + .expect("route succeeds"); + assert_eq!(route["provider"], "real"); + assert_eq!(route["candidates"], json!(["real"])); + + // pinning the preview as the explicit provider executes on that provider. + let (writer_ref, _frames, pump) = consumer_channel(&consumer).await; + let res = consumer + .trigger(TriggerRequest { + function_id: "router::chat".into(), + payload: json!({ + "writer_ref": writer_ref, + "model": "live-1", + "provider": route["provider"], + "messages": [] + }), + action: None, + timeout_ms: Some(30_000), + }) + .await + .expect("chat succeeds"); + assert_eq!(res["ok"], true, "chat response: {res}"); + assert_eq!(res["provider"], route["provider"]); + let _ = tokio::time::timeout(Duration::from_secs(5), pump).await; + + // an unrouteable model throws the same typed code the chat path throws. + let err = call(&consumer, "router::route", json!({ "model": "ghost" })) + .await + .expect_err("ghost model cannot route"); + assert_eq!(remote_code(&err), "router/no_provider_for_model"); + + consumer.shutdown(); + router_iii.shutdown(); +} + #[tokio::test(flavor = "multi_thread")] async fn consumer_cancellation_propagates_to_the_provider() { let engine = engine_or_skip!();