diff --git a/bin/ocx.mjs b/bin/ocx.mjs index c7f847542d7..dbf63b4ae2f 100755 --- a/bin/ocx.mjs +++ b/bin/ocx.mjs @@ -591,17 +591,30 @@ function runPackageManagerSelfUpdate(manager) { let stopAttempted = false; function recoverStoppedRuntimeAfterFailure(reason) { - const recoveryOwnership = readOwnership(); - const recoveryLiveness = currentPackageRuntimeLiveness(); - const recovery = planStoppedRuntimeRecovery({ - stopAttempted, - ...recoveryOwnership, - sameOwner: ownershipIdentity(recoveryOwnership) === stoppedOwnershipIdentity, - liveness: recoveryLiveness, - serviceInstalled: serviceWasInstalled, - launcherUsable: postUpdateLauncherUsable, - hadRuntimeState: hasRuntimeState, - }); + const planRecovery = () => { + const recoveryOwnership = readOwnership(); + const liveness = currentPackageRuntimeLiveness(); + return { + liveness, + plan: planStoppedRuntimeRecovery({ + stopAttempted, + ...recoveryOwnership, + sameOwner: ownershipIdentity(recoveryOwnership) === stoppedOwnershipIdentity, + liveness, + serviceInstalled: serviceWasInstalled, + launcherUsable: postUpdateLauncherUsable, + hadRuntimeState: hasRuntimeState, + }), + }; + }; + let { liveness: recoveryLiveness, plan: recovery } = planRecovery(); + if (recovery.action === "service") { + // The service manager starts the proxy outside this process tree, so it cannot join this + // lease, and holding the lease through the repair's health wait keeps that proxy from + // starting (#5760). Release it as the successful path does, then decide again. + releaseUpdateLease(); + ({ liveness: recoveryLiveness, plan: recovery } = planRecovery()); + } if (recovery.reason === "ownership-unknown") { console.error(`opencodex: ${reason}; runtime ownership is unknown, so automatic recovery was refused. Run 'ocx status --json' and repair the service-state record before retrying.`); } else if (recovery.reason === "ownership-transferred") { diff --git a/docs-site/src/content/docs/guides/integrations.md b/docs-site/src/content/docs/guides/integrations.md index 71549cf66e1..3a412061047 100644 --- a/docs-site/src/content/docs/guides/integrations.md +++ b/docs-site/src/content/docs/guides/integrations.md @@ -28,6 +28,13 @@ Generated catalogs include only enabled models from each provider selection. Thi downloads and managed integrations, including Pi and Aside. The management model list still shows the full roster so you can enable additional models. +`ocx uninstall` disables recorded integrations, including all owned Aside profiles, before deleting +OpenCodex's recovery state. Unreadable ownership, missing profile registration or a conflicting edit +stops that deletion. Cleanup is sequential: earlier successful disables are not undone when a later +one fails. If compensation also fails, a client file may be left in an intermediate state. Inspect +the reported client files and retained recovery snapshots before retrying; retained state does not +mean every client was restored or left unchanged. + For Gajae built-in presets, keep the routing choice in `~/.gjc/agent/config.yml`: ```yaml diff --git a/docs-site/src/content/docs/guides/remote-workspace.md b/docs-site/src/content/docs/guides/remote-workspace.md index 33db5f4eace..84ab94e12b0 100644 --- a/docs-site/src/content/docs/guides/remote-workspace.md +++ b/docs-site/src/content/docs/guides/remote-workspace.md @@ -27,6 +27,16 @@ lifecycle owner can retain cleanup authority through cancellation. Missing comma falls back to executing on the Hub. ::: +## RPC compatibility and timeouts + +Remote Workspace uses encrypted RPC v2. The Hub and every Executor must support v2; RPC v1 peers +fail closed instead of falling back to immediate execution, so upgrade the Hub and Executors +together. + +A timeout requests executor cancellation but does not confirm it. A grant may already be in transit, +or its operation may already be running. The default RPC timeout is 65 seconds, and `timeoutMs` +accepts inclusive values from 1 through 120,000 milliseconds. + ## Set up the Hub Computer 1 owns every coding-agent login and model session. Install and log in to whichever agents diff --git a/scripts/build-standalone.ts b/scripts/build-standalone.ts index f75fcc32251..fdbed98e128 100644 --- a/scripts/build-standalone.ts +++ b/scripts/build-standalone.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join, resolve } from "node:path"; +import { join, resolve, basename } from "node:path"; import { isStandaloneTarget, standaloneExecutableName } from "./standalone-targets"; function hostTarget(): string { @@ -28,17 +28,61 @@ if (!existsSync(join(guiDist, "index.html"))) { const output = resolve(argumentValue("--out") ?? join(repoRoot, "dist", "standalone", target)); mkdirSync(output, { recursive: true }); const executable = join(output, standaloneExecutableName(target)); -const result = Bun.spawnSync([ - process.execPath, - "build", - "--compile", - "--target", - target, - join(repoRoot, "src", "cli", "index.ts"), - "--outfile", - executable, -], { stdout: "inherit", stderr: "inherit" }); -if (result.exitCode !== 0) process.exit(result.exitCode); + +// Pre-bundle worker entrypoints so compiled binaries can spawn them from Blob +// URLs: oven-sh/bun#29124 breaks nested worker entrypoints resolved from +// $bunfs, so `new Worker(new URL(...))` dies with ModuleNotFound otherwise. +const WORKER_ENTRIES: Record = { + "policy-worker": join(repoRoot, "src", "storage", "policy-worker.ts"), + "restore-worker": join(repoRoot, "src", "storage", "restore-worker.ts"), + "history-worker": join(repoRoot, "src", "codex", "history-worker.ts"), +}; +const GEN_FILE = join(repoRoot, "src", "generated", "worker-bundles.gen.ts"); +const GEN_PLACEHOLDER = `// Generated by scripts/build-standalone.ts — do not edit by hand.\n// Placeholder for source checkouts; standalone builds overwrite this file\n// with pre-bundled worker sources before compiling.\nexport const WORKER_BUNDLES: Record = {};\n`; +const bundleLines: string[] = []; +const workerOut = join(output, ".worker-bundles"); +for (const [key, entry] of Object.entries(WORKER_ENTRIES)) { + const bundled = Bun.spawnSync([process.execPath, "build", entry, "--target", "bun", "--outdir", workerOut], { + stdout: "inherit", + stderr: "inherit", + }); + if (bundled.exitCode !== 0) process.exit(bundled.exitCode ?? 1); + const name = `${basename(entry, ".ts")}.js`; + bundleLines.push(` ${JSON.stringify(key)}: ${JSON.stringify(readFileSync(join(workerOut, name), "utf8"))},`); +} +writeFileSync( + GEN_FILE, + `// Generated by scripts/build-standalone.ts — do not edit by hand.\nexport const WORKER_BUNDLES: Record = {\n${bundleLines.join("\n")}\n};\n`, +); + +// The generated bundles only exist while the compile below consumes them. +// Always restore the empty placeholder afterwards — success or failure — so +// source-checkout runs and tests keep spawning workers from source files +// instead of a stale committed bundle. +function restoreGenPlaceholder(): void { + writeFileSync(GEN_FILE, GEN_PLACEHOLDER); +} + +const compileArgs = [process.execPath, "build", "--compile"]; +// Cross-compiling to the host target produces a binary the kernel kills on +// launch; only pass --target when it differs from the host. +if (target !== hostTarget()) compileArgs.push("--target", target); +compileArgs.push(join(repoRoot, "src", "cli", "index.ts"), "--outfile", executable); +// process.exit() skips `finally`, so exit only after the placeholder is back. +let compileExitCode: number; +try { + compileExitCode = Bun.spawnSync(compileArgs, { stdout: "inherit", stderr: "inherit" }).exitCode ?? 1; +} finally { + restoreGenPlaceholder(); +} +if (compileExitCode !== 0) process.exit(compileExitCode); + +// bun's ad-hoc linker signature does not always cover the embedded payload; +// macOS kills the executable on launch (SIGKILL) unless it is re-signed. +if (process.platform === "darwin") { + const sign = Bun.spawnSync(["codesign", "--force", "--sign", "-", executable], { stdout: "inherit", stderr: "inherit" }); + if (sign.exitCode !== 0) process.exit(sign.exitCode ?? 1); +} cpSync(guiDist, join(output, "gui", "dist"), { recursive: true }); const digest = createHash("sha256").update(readFileSync(executable)).digest("hex"); diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 57d1f3d4ab5..35a7e5f60ac 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -177,6 +177,8 @@ "standalone-build-script.test.ts": "gui", "standalone-service.test.ts": "service", "standalone.test.ts": "lib", + "worker-embed.test.ts": "lib", + "worker-embed-coverage.test.ts": "lib", "server-combo-held-response.test.ts": "server", "key-attribution.test.ts": "usage", "jev-stats.test.ts": "usage", @@ -1828,6 +1830,9 @@ "warmup-registration.test.ts": "ci-workflows", "warmup.test.ts": "codex-integration", "web-search-anthropic.test.ts": "web-search", + "devin-web-search.test.ts": "web-search", + "web-search-recovery-kind.test.ts": "web-search", + "chat-legacy-functions-combo.test.ts": "responses", "web-search-backend-union.test.ts": "web-search", "web-search-bridge-replay.test.ts": "web-search", "web-search-candidates.test.ts": "web-search", diff --git a/src/chat/inbound.ts b/src/chat/inbound.ts index 8630cc0dd79..c0d0947284a 100644 --- a/src/chat/inbound.ts +++ b/src/chat/inbound.ts @@ -211,6 +211,27 @@ function toolCallsToItems( } } +function legacyFunctionCallToItem( + value: unknown, + input: Rec[], + knownNameByCallId: Map, + awaitingToolResult: Set, + sequence: number, +): { callId: string; name: string } | null { + if (value === undefined) return null; + if (!isRec(value) || typeof value.name !== "string" || value.name.length === 0) { + throw new ChatCompletionsRequestError("assistant function_call requires a name"); + } + const args = typeof value.arguments === "string" + ? value.arguments + : JSON.stringify(value.arguments ?? {}); + const callId = `call_legacy_${String(sequence).padStart(4, "0")}`; + knownNameByCallId.set(callId, value.name); + awaitingToolResult.add(callId); + input.push({ type: "function_call", call_id: callId, name: value.name, arguments: args }); + return { callId, name: value.name }; +} + function toolsToResponses(tools: unknown): Rec[] | undefined { if (!Array.isArray(tools) || tools.length === 0) return undefined; const out: Rec[] = []; @@ -243,6 +264,24 @@ function toolsToResponses(tools: unknown): Rec[] | undefined { return out.length > 0 ? out : undefined; } +function legacyFunctionsToResponses(functions: unknown): Rec[] | undefined { + if (functions === undefined) return undefined; + if (!Array.isArray(functions)) throw new ChatCompletionsRequestError("functions must be an array"); + const out: Rec[] = []; + for (const raw of functions) { + if (!isRec(raw) || typeof raw.name !== "string" || raw.name.length === 0) { + throw new ChatCompletionsRequestError("functions entries require a name"); + } + out.push({ + type: "function", + name: raw.name, + ...(typeof raw.description === "string" ? { description: raw.description } : {}), + ...(isRec(raw.parameters) ? { parameters: raw.parameters } : {}), + }); + } + return out.length > 0 ? out : undefined; +} + function toolChoiceToResponses(choice: unknown, body: Rec): void { if (choice === undefined || choice === null) return; if (choice === "auto" || choice === "none" || choice === "required") { @@ -269,6 +308,18 @@ function toolChoiceToResponses(choice: unknown, body: Rec): void { } } +function legacyFunctionChoiceToResponses(choice: unknown, body: Rec): void { + if (choice === undefined || choice === null) return; + if (choice === "auto" || choice === "none") { + body.tool_choice = choice; + return; + } + if (!isRec(choice) || typeof choice.name !== "string" || choice.name.length === 0) { + throw new ChatCompletionsRequestError("function_call requires auto, none, or a function name"); + } + body.tool_choice = { type: "function", name: choice.name }; +} + /** * Chat Completions nests the subset under `allowed_tools`, Responses carries `mode`/`tools` * on the choice itself, and each entry names its tool under a member keyed by its own type @@ -389,6 +440,8 @@ export function chatCompletionsToResponsesBody(raw: unknown): Rec { // Recover replace-style tool calls incrementally instead of rebuilding the // call-id index from the entire translated transcript for every message. const knownNameByCallId = new Map(); + const legacyAwaiting: Array<{ callId: string; name: string }> = []; + let legacyCallSequence = 0; // Tool calls whose result has not arrived yet. Several adapters need a call and its output // to stay adjacent — Kiro refuses an interrupted pair (src/adapters/kiro/payload.ts) and the // Anthropic and Google mappers synthesize a missing result — so an instruction that arrives @@ -405,6 +458,7 @@ export function chatCompletionsToResponsesBody(raw: unknown): Rec { const beginConversationTurn = (): void => { releaseHeldInstructions(); awaitingToolResult.clear(); + legacyAwaiting.length = 0; }; for (const msg of raw.messages) { @@ -462,6 +516,16 @@ export function chatCompletionsToResponsesBody(raw: unknown): Rec { if (msg.tool_calls !== undefined) { toolCallsToItems(msg.tool_calls, input, knownNameByCallId, awaitingToolResult); } + if (msg.function_call !== undefined && msg.function_call !== null) { + const call = legacyFunctionCallToItem( + msg.function_call, + input, + knownNameByCallId, + awaitingToolResult, + ++legacyCallSequence, + ); + if (call) legacyAwaiting.push(call); + } break; } case "function": { @@ -472,6 +536,17 @@ export function chatCompletionsToResponsesBody(raw: unknown): Rec { "Legacy function-result image translation is not implemented. Use tool_calls and role:tool with tool_call_id.", ); } + const name = typeof msg.name === "string" ? msg.name : ""; + if (!name) throw new ChatCompletionsRequestError("function messages require a name"); + const pendingIndex = legacyAwaiting.findIndex(call => call.name === name); + if (pendingIndex < 0) { + throw new ChatCompletionsRequestError(`function result has no pending call named ${name}`); + } + const [call] = legacyAwaiting.splice(pendingIndex, 1); + const output = contentToText(msg.content); + input.push({ type: "function_call_output", call_id: call!.callId, output }); + awaitingToolResult.delete(call!.callId); + if (awaitingToolResult.size === 0) releaseHeldInstructions(); break; } case "tool": { @@ -507,9 +582,13 @@ export function chatCompletionsToResponsesBody(raw: unknown): Rec { if (systemParts.length > 0) body.instructions = systemParts.join("\n\n"); - const tools = toolsToResponses(raw.tools); - if (tools) body.tools = tools; - toolChoiceToResponses(raw.tool_choice, body); + const tools = [ + ...(toolsToResponses(raw.tools) ?? []), + ...(legacyFunctionsToResponses(raw.functions) ?? []), + ]; + if (tools.length > 0) body.tools = tools; + if (raw.tool_choice !== undefined) toolChoiceToResponses(raw.tool_choice, body); + else legacyFunctionChoiceToResponses(raw.function_call, body); const maxTokens = typeof raw.max_completion_tokens === "number" ? raw.max_completion_tokens diff --git a/src/chat/outbound.ts b/src/chat/outbound.ts index 817aa11b7fb..8e1b684861b 100644 --- a/src/chat/outbound.ts +++ b/src/chat/outbound.ts @@ -154,9 +154,11 @@ function appendedUtf8Bytes(previous: string, previousBytes: number, fragment: st const fragmentFirst = fragment.charCodeAt(0); if (previousLast >= 0xd800 && previousLast <= 0xdbff && fragmentFirst >= 0xdc00 && fragmentFirst <= 0xdfff) { - // Buffer.byteLength() replaces each isolated surrogate with three bytes, while the joined - // pair is one four-byte scalar. Preserve full-string sizing without re-encoding the prefix. - nextBytes -= 2; + // Buffer implementations disagree on the encoded size of an isolated surrogate. Measure + // the join delta so incremental accounting equals the completed scalar on every runtime. + const tail = previous[previous.length - 1]!; + const head = fragment[0]!; + nextBytes += Buffer.byteLength(tail + head) - Buffer.byteLength(tail) - Buffer.byteLength(head); } return nextBytes; } diff --git a/src/cli/uninstall-client-state.ts b/src/cli/uninstall-client-state.ts index af45c58573e..f200bf11b1f 100644 --- a/src/cli/uninstall-client-state.ts +++ b/src/cli/uninstall-client-state.ts @@ -6,6 +6,7 @@ import { inspectRemoteDesktopCleanup, readDesktopDisconnectReceipt } from "../cl import { removeOwnedConfigState, type ConfigRemovalResult } from "../lib/config-ownership"; import { windowsSecretAclReapPendingAtOrBelow } from "../lib/windows-secret-acl"; import { sharedTeardownAuthorized, type UninstallObservation } from "./uninstall-plan"; +import { cleanupOwnedIntegrationsBeforeUninstall } from "./uninstall-integrations"; export interface UninstallClientStateDeps { readConnection: typeof readClientConnectionState; @@ -13,6 +14,7 @@ export interface UninstallClientStateDeps { readReceipt: typeof readDesktopDisconnectReceipt; disconnect: (options?: Parameters[0]) => Promise; withLifecycle: typeof withClientLifecycle; + cleanupIntegrations: typeof cleanupOwnedIntegrationsBeforeUninstall; remove: () => ConfigRemovalResult; /** True while a timed-out icacls child still owns a path at or below the config directory. */ aclReapPending: (rootPath: string) => boolean; @@ -24,6 +26,7 @@ const defaults: UninstallClientStateDeps = { readReceipt: readDesktopDisconnectReceipt, disconnect: options => disconnectClient(options), withLifecycle: withClientLifecycle, + cleanupIntegrations: cleanupOwnedIntegrationsBeforeUninstall, remove: () => removeOwnedConfigState(getConfigDir()), aclReapPending: rootPath => windowsSecretAclReapPendingAtOrBelow(rootPath), }; @@ -85,6 +88,9 @@ export async function removeOwnedConfigAfterDesktopCleanup( if (deps.aclReapPending(getConfigDir())) { throw new Error("Client cleanup refused: ACL hardening still owns a path under the config directory."); } + // Integration records are the authority that permits removing only OpenCodex-owned fragments + // from third-party files. Delete them only after every recorded contribution is retired. + await deps.cleanupIntegrations(); return deps.remove(); }); } diff --git a/src/cli/uninstall-integrations.ts b/src/cli/uninstall-integrations.ts new file mode 100644 index 00000000000..5645afee2e2 --- /dev/null +++ b/src/cli/uninstall-integrations.ts @@ -0,0 +1,115 @@ +import type { ExportModel } from "../clients/config-export"; +import { guardAsideProfileIO, listAsideProfiles } from "../clients/aside-profiles"; +import { loadConfig } from "../config"; +import { listAsideProfileStores } from "../integrations/aside-profile-context"; +import { isIntegrationClientId, type IntegrationClientId } from "../integrations/registry"; +import { createIntegrationStateStore, type IntegrationStateStore } from "../integrations/store"; +import { disableIntegrationCoordinated, type WriteOutcome } from "../integrations/writer"; +import { loadExportModels } from "../server/management/model-rows"; +import type { OcxConfig } from "../types"; + +export interface UninstallIntegrationCleanupDeps { + createStore: () => IntegrationStateStore; + loadConfig: () => OcxConfig; + loadModels: (config: OcxConfig) => Promise; + disable: (input: Parameters[0]) => Promise; + env?: NodeJS.ProcessEnv; + home?: string; +} + +const defaults: UninstallIntegrationCleanupDeps = { + createStore: () => createIntegrationStateStore(), + loadConfig, + loadModels: config => loadExportModels(config), + disable: input => disableIntegrationCoordinated(input), +}; + +export interface UninstallIntegrationCleanupResult { + attempted: number; + changed: number; +} + +/** + * Remove every contribution we can still prove we own before uninstall deletes that proof. + * A single refusal aborts config removal: preserving recovery state is safer than leaving an + * external client pointed at a proxy that no longer exists. + */ +export async function cleanupOwnedIntegrationsBeforeUninstall( + deps: UninstallIntegrationCleanupDeps = defaults, +): Promise { + const store = deps.createStore(); + const records = store.readRecordsStrict(); + const rawIds = Object.keys(records); + for (const id of rawIds) { + if (!isIntegrationClientId(id)) { + throw new Error(`integration cleanup refused: ownership names unknown client ${id}`); + } + } + type CleanupTarget = Pick[0], "clientId" | "store" | "io" | "resolvedPaths"> + & { store: IntegrationStateStore; profileId?: number }; + const targets: CleanupTarget[] = (rawIds as IntegrationClientId[]).sort() + .map(clientId => ({ clientId, store })); + // Read every child before any mutation. A broken child must not look like no ownership. + for (const child of listAsideProfileStores(store)) { + const childRecords = child.store.readRecordsStrict(); + if (Object.keys(childRecords).some(id => id !== "aside")) { + throw new Error("integration cleanup refused: Aside profile storage names another client"); + } + if (childRecords.aside) targets.push({ clientId: "aside", ...child }); + } + if (targets.length === 0) return { attempted: 0, changed: 0 }; + + const asideTargets = targets.filter(target => target.clientId === "aside"); + if (asideTargets.length > 0) { + const profiles = listAsideProfiles(deps.env, deps.home); + const claimedProfiles = new Set(); + for (const target of asideTargets) { + const record = target.store.readRecordsStrict().aside; + const profile = profiles.find(candidate => candidate.configPath === record?.configPath + && (target.profileId === undefined || target.profileId === candidate.id)); + if (!profile || claimedProfiles.has(profile.id)) { + throw new Error("integration cleanup refused: Aside profile ownership is missing or mismatched"); + } + claimedProfiles.add(profile.id); + target.resolvedPaths = { configPath: profile.configPath, detectDir: profile.detectDir }; + // io() closes over the raw store; route bookkeeping back through its guarded facade. + target.io = guardAsideProfileIO(profile, { + ...target.store.io(), + appendJournal: entry => target.store.appendJournal(entry), + putRecord: record => target.store.putRecord(record), + dropRecord: clientId => target.store.dropRecord(clientId), + }, profiles); + } + } + + const config = deps.loadConfig(); + const models = await deps.loadModels(config); + let changed = 0; + for (const { clientId, store: targetStore, io, resolvedPaths } of targets) { + const result = await deps.disable({ + clientId, + models, + config, + port: config.port, + store: targetStore, + ...(io ? { io } : {}), + ...(resolvedPaths ? { resolvedPaths } : {}), + ...(deps.env ? { env: deps.env } : {}), + ...(deps.home ? { home: deps.home } : {}), + }); + if (!result.ok) { + throw new Error(`integration cleanup refused for ${clientId}: ${result.message}` + + (result.residual ? "; recovery did not complete; inspect the client file and retained snapshots before retrying" : "")); + } + if (targetStore.readRecordsStrict()[clientId]) { + throw new Error(`integration cleanup did not retire ownership for ${clientId}`); + } + if (result.changed) changed++; + } + // A profile can gain ownership while catalog loading or an earlier disable awaited its lock. + if ([store, ...listAsideProfileStores(store).map(child => child.store)] + .some(current => Object.keys(current.readRecordsStrict()).length > 0)) { + throw new Error("integration cleanup refused: ownership changed before removal"); + } + return { attempted: targets.length, changed }; +} diff --git a/src/codex/history-job.ts b/src/codex/history-job.ts index 134550f478c..ea0a62d175f 100644 --- a/src/codex/history-job.ts +++ b/src/codex/history-job.ts @@ -24,6 +24,7 @@ import type { HistoryWorkerResult, } from "./history-worker"; import { currentHistoryDbBusyTimeoutMs, resolveExistingHistoryBackupPath } from "./history-provider"; +import { spawnWorker } from "../lib/worker-embed"; import type { CodexHistoryFailureReason, CodexHistoryVerifiedNoopProof } from "./history-provider"; import { getCodexHome, resolveCodexStateDbPath } from "./paths"; @@ -345,7 +346,7 @@ export async function runCodexHistoryJob( let worker: Worker; try { - worker = new Worker(new URL("./history-worker.ts", import.meta.url).href); + worker = spawnWorker(new URL("./history-worker.ts", import.meta.url).href, "history-worker"); } catch (error) { return { kind: "failed", diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index 492eb1ac270..f300e493a67 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -204,6 +204,8 @@ export function pickComboTarget( exclude?: Iterable; eligible?: (target: NormalizedComboTarget) => boolean; now?: number; + /** Inspect a round-robin choice without mutating its sticky/weight state. */ + preview?: boolean; } = {}, ): ComboPick | null { const writerGeneration = captureConfigGeneration(); @@ -222,7 +224,13 @@ export function pickComboTarget( let state = selectionState.get(comboId); if (!state) { state = { successes: 0, currentWeights: new Map(), successfulUses: new Map() }; - selectionState.set(comboId, state); + if (!options.preview) selectionState.set(comboId, state); + } else if (options.preview) { + state = { + ...state, + currentWeights: new Map(state.currentWeights), + successfulUses: new Map(state.successfulUses), + }; } if (state.activeKey) { targetIndex = combo.targets.findIndex(target => targetKey(target) === state.activeKey && eligible(target)); @@ -542,11 +550,11 @@ export function clearComboSelectionState(comboId?: string): void { selectionState.delete(comboId); } -export function tryPickComboModel(config: OcxConfig, modelId: string): ComboPick | null { +export function tryPickComboModel(config: OcxConfig, modelId: string, preview = false): ComboPick | null { const comboId = resolveComboId(config, modelId); if (!comboId) return null; if (!getCombo(config, comboId)) throw new UnknownComboError(comboId); - const picked = pickComboTarget(config, comboId); + const picked = pickComboTarget(config, comboId, { preview }); if (!picked) throw new NoAvailableComboTargetsError(comboId); return picked; } diff --git a/src/generated/worker-bundles.gen.ts b/src/generated/worker-bundles.gen.ts new file mode 100644 index 00000000000..a1a928011a7 --- /dev/null +++ b/src/generated/worker-bundles.gen.ts @@ -0,0 +1,4 @@ +// Generated by scripts/build-standalone.ts — do not edit by hand. +// Placeholder for source checkouts; standalone builds overwrite this file +// with pre-bundled worker sources before compiling. +export const WORKER_BUNDLES: Record = {}; diff --git a/src/images/loop.ts b/src/images/loop.ts index c09a07336b5..dd12fbb9d48 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -325,12 +325,19 @@ export interface ImageBridgeDeps { * `retryParsed` is the exact iteration-local request the retry will be built from. The loop * sends a shallow copy of the outer parsed request, so a rotation that rebinds only the outer * object never reaches the wire. Optional so existing callers keep compiling. + * + * A rotation may cross ACCOUNTS, not just keys, and the attempt row is where an operator reads + * which one happened. A rotator that knows which kind it performed returns it alongside the + * adapter -- the same `{ adapter, recoveryKind }` shape `onCredentialError` already uses below. */ on429?: ( retryAfterHeader: string | null, responseHeaders?: Headers, retryParsed?: OcxParsedRequest, - ) => ProviderAdapter | null | Promise; + ) => + | { adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } + | null + | Promise<{ adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null>; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; /** Called when the bridged Responses stream completes (parity with runTurn / routed paths). */ @@ -668,9 +675,9 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise {}); } catch { /* already closed */ } - adapter = rotated; + adapter = rotated.adapter; yield { type: "heartbeat" }; - prepared = await fetchOnce(adapter, "key-429"); + prepared = await fetchOnce(adapter, rotated.recoveryKind); } // Final headers have arrived. Clear only the deadline timer before ANY body read. diff --git a/src/integrations/aside-profile-context.ts b/src/integrations/aside-profile-context.ts index 65ee091c06f..f5091b4b549 100644 --- a/src/integrations/aside-profile-context.ts +++ b/src/integrations/aside-profile-context.ts @@ -131,6 +131,23 @@ export function asideRootStore(input: AsideProfilesInput): IntegrationStateStore return guardedStore(raw, raw.root); } +/** Enumerate persisted owners, not desired/current profiles: uninstall must retain orphaned proof. */ +export function listAsideProfileStores(rootStore: IntegrationStateStore): Array<{ profileId: number; store: IntegrationStateStore }> { + const directory = join(rootStore.root, "aside-profiles"); + storeGuard(rootStore.root, directory)(); + let names: string[]; + try { names = readdirSync(directory); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + return storeUnsafe(); + } + return names.sort().map(name => { + const profileId = Number(name); + if (!Number.isSafeInteger(profileId) || profileId < 0 || String(profileId) !== name) storeUnsafe(); + return { profileId, store: guardedStore(createIntegrationStateStore(join(directory, name)), rootStore.root) }; + }); +} + export function assertAsideSnapshotEntry(entry: JournalEntry): void { if (!entry || entry.clientId !== "aside" || typeof entry.opId !== "string" || !/^[A-Za-z0-9_-]{1,128}$/.test(entry.opId) || !entry.snapshot || !["none", "stored", "expired"].includes(entry.snapshot.kind) diff --git a/src/lab/fabric/executor.ts b/src/lab/fabric/executor.ts index 66c9e2eb25f..d948d970502 100644 --- a/src/lab/fabric/executor.ts +++ b/src/lab/fabric/executor.ts @@ -15,8 +15,16 @@ import { SYNTHETIC_VALUE_PATH, } from "./constants"; import { applySyntheticPatch, parseSyntheticPatchV1 } from "./patch"; -import { fabricProducerIsolationLimits, runIsolatedFabricProducer } from "./producer-isolate"; -import { assertNotUnderUserRepo, createSyntheticScratch, type ScratchTree } from "./scratch"; +import { + fabricProducerIsolationLimits, + isUnconfirmedProducerTermination, + runIsolatedFabricProducer, +} from "./producer-isolate"; +import { + assertNotUnderUserRepo, + createSyntheticScratch, + type ScratchTree, +} from "./scratch"; import { buildTaskSubjectV1, sandboxProfileDigest, @@ -24,6 +32,7 @@ import { taskSubjectId, verifierManifestDigest, } from "./subject"; + import type { FabricExecutionAuthority, FabricHarnessProducerKind, @@ -183,6 +192,9 @@ async function runFabricSyntheticPatchTaskInternal(input: { let scratch: ScratchTree | undefined; let producerCompletedAt = startedAt; let lastActivityAt = startedAt; + // Rejected while the producer child might still be running: scratch must not + // be removed under it; absence of a termination proof requires manual review. + let producerTerminationUnconfirmed = false; try { scratch = createSyntheticScratch(input.configDir); @@ -236,6 +248,7 @@ async function runFabricSyntheticPatchTaskInternal(input: { const completedAt = input.now?.() ?? Date.now(); usage.elapsedMs = completedAt - startedAt; usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); + producerTerminationUnconfirmed = isUnconfirmedProducerTermination(error); if (error instanceof FabricTaskError) { const failure = failureFromError(error); return { @@ -418,7 +431,16 @@ async function runFabricSyntheticPatchTaskInternal(input: { }), }; } finally { - scratch?.cleanup(); + if (scratch) { + if (producerTerminationUnconfirmed) { + // No writes into producer-controlled scratch after uncertain termination. + // Age and later pipe closure cannot prove every descendant has exited. + // Retain the tree, including across parent exit and later task creation. + console.warn("[lab] Producer termination unconfirmed; scratch retained for manual review."); + } else { + scratch.cleanup(); + } + } } } diff --git a/src/lab/fabric/producer-isolate.ts b/src/lab/fabric/producer-isolate.ts index 70291adffc0..aab2a25d597 100644 --- a/src/lab/fabric/producer-isolate.ts +++ b/src/lab/fabric/producer-isolate.ts @@ -15,6 +15,25 @@ import { FabricTaskError } from "./types"; const CHILD_ENTRY = join(dirname(fileURLToPath(import.meta.url)), "producer-child.ts"); +/** + * Bounded drain window between a child's `exit` and our decision. `close` also + * waits for the child's stdio to end, and a descendant holding an inherited pipe + * can delay it forever — so a missing `close` must not keep the run pending. + * The bound only has to cover an ordinary stdio flush after process death: the + * kernel releases a dead child's pipe ends immediately, so a `close` slower + * than this almost always means a descendant still holds a pipe. A slow-but- + * normal pipe that outlives the bound costs an inconclusive verdict plus + * deferred scratch cleanup — a bounded price that does not grow with the wait. + */ +const EXIT_DRAIN_MS = 250; + +/** + * Bounded wait for exit/close after a parent-owned SIGKILL. Neither event is + * guaranteed — a child in uninterruptible sleep, or a kill() that failed, produces + * neither — and a latched killReason must not leave the run pending forever. + */ +const KILL_CONFIRM_MS = 2_000; + type FabricProducerIsolationLimits = { totalTimeoutMs: number; inactivityTimeoutMs: number; @@ -80,6 +99,24 @@ function killChild(child: ChildProcess): void { } } +/** Whether the producer rejected while its child might still be running. */ +export function isUnconfirmedProducerTermination(error: unknown): boolean { + return error instanceof FabricTaskError + && (error as FabricTaskError & { unconfirmedTermination?: boolean }).unconfirmedTermination === true; +} + +/** + * The release signal attached to an unconfirmed-termination rejection: resolves + * once every monitored inherited stdio pipe reports its natural close. This is + * observation only: closing a pipe does not prove a descendant exited and must + * not authorize deletion. Undefined when nothing monitorable remained. + */ +export function producerTerminationSignal(error: unknown): Promise | undefined { + if (!error || typeof error !== "object") return undefined; + const signal = (error as { stdioRelease?: unknown }).stdioRelease; + return signal instanceof Promise ? signal : undefined; +} + /** Run a fabric patch producer in an isolated child process with parent-owned timeouts. */ export async function runIsolatedFabricProducer(request: IsolateRequest): Promise { const now = request.now ?? (() => Date.now()); @@ -110,8 +147,37 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis let stderrBytes = 0; let settled = false; let childClosed = false; + let childExitedAt: number | undefined; let receivedResult: SyntheticPatchV1 | undefined; let killReason: FabricTaskError | undefined; + let reapTimer: ReturnType | undefined; + let killWatchdog: ReturnType | undefined; + // Resolves once every still-open inherited stdio pipe reports its natural + // close. This is not a writer-termination proof or scratch deletion lease. + // Undefined when no open pipe can be monitored. + let stdioReleaseSignal: Promise | undefined; + + // Keep monitorable pipes open but unref'd so they never extend process + // lifetime; a stream without unref() is destroyed instead, and its + // self-inflicted close must not count toward the release signal. + const armStdioRelease = (): void => { + const waiters: Promise[] = []; + let unmonitorable = false; + for (const stream of [child.stdout, child.stderr]) { + if (!stream || stream.destroyed) continue; + const unref = (stream as unknown as { unref?: unknown }).unref; + if (typeof unref === "function") { + waiters.push(new Promise((resolve) => stream.once("close", resolve))); + unref.call(stream); + } else { + unmonitorable = true; + try { stream.destroy(); } catch { /* already closed */ } + } + } + if (waiters.length > 0 && !unmonitorable) { + stdioReleaseSignal = Promise.all(waiters).then(() => undefined); + } + }; const finish = (fn: () => void) => { // A latched failure owns settlement, but scratch cleanup must wait for close. @@ -119,6 +185,8 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis settled = true; clearTimeout(totalTimer); clearTimeout(inactivityTimer); + if (reapTimer) clearTimeout(reapTimer); + if (killWatchdog) clearTimeout(killWatchdog); if (killReason) reject(killReason); else fn(); }; @@ -127,7 +195,31 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis if (settled || killReason) return; killReason = error; if (childClosed) finish(() => reject(error)); - else killChild(child); + else { + killChild(child); + // SIGKILL does not guarantee exit/close: an uninterruptible child, or a + // kill() that failed, emits neither, and the latched killReason would + // otherwise keep this run pending forever. Bound the wait; on expiry + // reject with the original reason flagged unconfirmed so the caller + // defers scratch cleanup instead of racing a child that may live. + killWatchdog = setTimeout(() => { + if (childClosed || settled) return; + settled = true; + clearTimeout(totalTimer); + clearTimeout(inactivityTimer); + if (reapTimer) clearTimeout(reapTimer); + if (killWatchdog) { + clearTimeout(killWatchdog); + killWatchdog = undefined; + } + armStdioRelease(); + try { child.unref(); } catch { /* fake children may lack unref */ } + const reason = killReason! as FabricTaskError & { unconfirmedTermination?: boolean; stdioRelease?: Promise }; + reason.unconfirmedTermination = true; + reason.stdioRelease = stdioReleaseSignal; + reject(killReason); + }, KILL_CONFIRM_MS); + } }; const expiredDeadline = (at: number): FabricTaskError | undefined => { @@ -163,7 +255,10 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis try { const message = parseProducerProtocolLine(line); if (message.type === "activity" || message.type === "result") { - const at = budgetNow(); + // Bytes drained after `exit` are judged at the exit timestamp: the + // process met its budgets when it died, so the drain must not + // condemn data it wrote while still inside them. + const at = childExitedAt ?? budgetNow(); const expired = expiredDeadline(at); if (expired) { settleTimeout(expired); @@ -172,13 +267,12 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis if (message.type === "activity") { lastActivityAt = request.now ? at : now(); inactivityDeadline = at + request.inactivityTimeoutMs; - armInactivity(); + if (childExitedAt === undefined) armInactivity(); return; } } if (message.type === "result") { receivedResult = message.patch; - finish(() => resolve({ patch: message.patch, lastActivityAt })); return; } if (message.type === "error") { @@ -193,8 +287,7 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis : "harness_failure"; const attribution = message.attribution === "environment" ? "environment" : "harness"; const fabricError = new FabricTaskError(message.message, fabricCode, attribution); - finish(() => reject(fabricError)); - killChild(child); + settleTimeout(fabricError); return; } } catch (error) { @@ -228,8 +321,7 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis }); child.stdout?.on("error", (error) => { - if (settled) return; - finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + settleTimeout(new FabricTaskError(error.message, "harness_failure", "harness")); }); child.stderr?.on("data", (chunk: Buffer | string) => { @@ -244,24 +336,32 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis }); child.on("error", (error) => { - finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + if (child.pid === undefined) { + // A spawn failure has no process to supervise and may never emit close. + childClosed = true; + finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + return; + } + settleTimeout(new FabricTaskError(error.message, "harness_failure", "harness")); }); child.stdin?.on("error", (error: NodeJS.ErrnoException) => { - if (settled || killReason || error.code === "EPIPE") return; - killChild(child); - finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + if (error.code === "EPIPE") return; + settleTimeout(new FabricTaskError(error.message, "harness_failure", "harness")); }); - child.on("close", (code, signal) => { - childClosed = true; + const decide = (code: number | null, signal: NodeJS.Signals | null, reaped = false) => { if (settled) return; if (killReason) { - finish(() => reject(killReason!)); - return; - } - if (receivedResult) { - finish(() => resolve({ patch: receivedResult!, lastActivityAt })); + const reason = killReason as FabricTaskError & { unconfirmedTermination?: boolean; stdioRelease?: Promise }; + if (reaped) { + // The kill was answered by exit but the pipes stayed open: a + // descendant can still hold them, so scratch cleanup must defer to + // the same release contract as an unconfirmed kill. + reason.unconfirmedTermination = true; + reason.stdioRelease = stdioReleaseSignal; + } + finish(() => reject(reason)); return; } if (stdoutBuffer.trim()) { @@ -272,15 +372,85 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis /* fall through */ } } - if (signal === "SIGKILL") { - finish(() => reject(new FabricTaskError("total timeout exceeded", "timeout", "environment"))); + if (reaped) { + // `close` never followed `exit`: the pipes outlived the producer, which + // may mean a descendant escaped supervision — but the drain also cannot + // rule out a stalled event loop or a slow pipe, so this is reported as + // an inconclusive harness failure rather than a sandbox escape. Either + // way the result is rejected: it must never resolve while a descendant + // might still be alive to mutate scratch after cleanup. The recorded + // exit status only narrows the message; the deferral contract is the + // same for a clean exit and a nonzero or signaled one. + const failure = new FabricTaskError( + code !== 0 || signal + ? `isolated producer exited (${code ?? signal ?? "unknown"}); its stdio never closed` + : "isolated producer exited but its stdio never closed", + "harness_failure", + "harness", + ) as FabricTaskError & { unconfirmedTermination?: boolean; stdioRelease?: Promise }; + // A descendant holding an inherited pipe can still use scratch after + // the direct child exited, so the caller must retain scratch for manual + // review — pipe closure alone cannot prove a descendant terminated. + failure.unconfirmedTermination = true; + failure.stdioRelease = stdioReleaseSignal; + finish(() => reject(failure)); return; } - finish(() => reject(new FabricTaskError( - code === 0 ? "isolated producer returned no result" : `isolated producer exited (${code ?? signal ?? "unknown"})`, - "harness_failure", - "harness", - ))); + // A stored result is accepted only when the child exited normally. A + // signaled or nonzero exit without a latched reason is a harness failure — + // parent-owned kills always carry a killReason, so this is never a timeout. + if (code !== 0 || signal) { + finish(() => reject(new FabricTaskError( + `isolated producer exited (${code ?? signal ?? "unknown"})`, + "harness_failure", + "harness", + ))); + return; + } + if (receivedResult) { + finish(() => resolve({ patch: receivedResult!, lastActivityAt })); + return; + } + finish(() => reject(new FabricTaskError("isolated producer returned no result", "harness_failure", "harness"))); + }; + + child.on("exit", (code, signal) => { + if (childClosed || settled) return; + // `exit` ends the budget window even while `close` is still pending on + // stdio: an already-met deadline still applies, and no producer code can + // breach one after this point, so both budget timers are disarmed now. + childExitedAt = budgetNow(); + if (!killReason) { + const expired = expiredDeadline(childExitedAt); + if (expired) killReason = expired; + } + clearTimeout(totalTimer); + clearTimeout(inactivityTimer); + // The direct child is confirmed dead, so the kill-confirmation watchdog + // is moot; only the stdio drain still needs its bound. + if (killWatchdog) { + clearTimeout(killWatchdog); + killWatchdog = undefined; + } + // The direct child is dead, but `close` also waits for its stdio to end. + // Bound the drain so a descendant holding an inherited pipe cannot keep + // the run pending, then settle from the recorded exit status. + reapTimer = setTimeout(() => { + if (reapTimer) { + clearTimeout(reapTimer); + reapTimer = undefined; + } + if (childClosed || settled) return; + childClosed = true; + armStdioRelease(); + decide(code, signal, true); + }, EXIT_DRAIN_MS); + }); + + child.on("close", (code, signal) => { + if (childClosed) return; + childClosed = true; + decide(code, signal); }); const payload = JSON.stringify({ @@ -299,8 +469,7 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis : undefined, }); if (Buffer.byteLength(payload, "utf8") > FABRIC_PRODUCER_REQUEST_MAX_BYTES) { - killChild(child); - finish(() => reject(new FabricTaskError("producer request exceeds protocol limit", "budget_exhausted", "environment"))); + settleTimeout(new FabricTaskError("producer request exceeds protocol limit", "budget_exhausted", "environment")); return; } @@ -308,13 +477,11 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis child.stdin?.write(payload); child.stdin?.end(); } catch (error) { - if (killReason) return; - killChild(child); - finish(() => reject(new FabricTaskError( + settleTimeout(new FabricTaskError( error instanceof Error ? error.message : String(error), "harness_failure", "harness", - ))); + )); return; } }); diff --git a/src/lab/fabric/scratch.ts b/src/lab/fabric/scratch.ts index 6f0c9e8a60d..a634df8022e 100644 --- a/src/lab/fabric/scratch.ts +++ b/src/lab/fabric/scratch.ts @@ -9,6 +9,7 @@ import { readdirSync, readSync, rmSync, + statSync, writeSync, type Stats, } from "node:fs"; @@ -30,6 +31,9 @@ interface TrustedScratchDir { identity: string; } +// Unconfirmed producer trees are retained for manual review. Neither a marker +// inside producer-writable scratch nor its age grants later deletion authority. + /** Require stats to describe a regular file, not a symlink or special node. */ function assertRegularFile(stats: Stats, label: string): void { if (!stats.isFile() || stats.isSymbolicLink() || stats.isDirectory() || stats.isFIFO() || stats.isSocket() || stats.isCharacterDevice() || stats.isBlockDevice()) { @@ -299,11 +303,14 @@ export function createSyntheticScratch(configDir?: string): ScratchTree { } finally { closeSync(fd); } - const trustedForCleanup = trusted; + // The pinned root is only needed to write the fixture; every later access + // opens its own trusted handle. Close it now so deferred cleanup — which + // may wait on a descendant-held pipe or never observe a release signal — + // cannot leak the descriptor (or hold the directory open on Windows). + closeTrustedScratchRoot(trusted); return { root, cleanup: () => { - closeTrustedScratchRoot(trustedForCleanup); try { rmSync(root, { recursive: true, force: true, maxRetries: 3 }); } catch { diff --git a/src/lib/worker-embed.ts b/src/lib/worker-embed.ts new file mode 100644 index 00000000000..7ff393c4822 --- /dev/null +++ b/src/lib/worker-embed.ts @@ -0,0 +1,33 @@ +/** + * Worker spawning that survives `bun build --compile`. + * + * Bun standalone executables fail to resolve nested worker entrypoints + * (oven-sh/bun#29124): the main bundle is flattened into /$bunfs/root while + * additional entrypoints keep their nested paths, so `new Worker(new URL(...))` + * dies with ModuleNotFound in compiled binaries. Standalone builds therefore + * embed each worker as a pre-bundled source string (generated by + * scripts/build-standalone.ts into src/generated/worker-bundles.gen.ts) and + * spawn it from a Blob URL, which works in both compiled binaries and source + * checkouts. Source checkouts without generated bundles fall back to the + * dev-mode URL form. + */ +import { WORKER_BUNDLES } from "../generated/worker-bundles.gen"; + +export function spawnWorker(devUrl: string, key: string): Worker { + const src = WORKER_BUNDLES[key]; + if (src) { + const blobUrl = URL.createObjectURL(new Blob([src], { type: "text/javascript" })); + let worker: Worker; + try { + worker = new Worker(blobUrl); + } catch (error) { + URL.revokeObjectURL(blobUrl); + throw error; + } + // Long-lived servers schedule workers repeatedly; release the blob URL + // once the thread exits instead of leaking one per run. + worker.addEventListener("close", () => URL.revokeObjectURL(blobUrl)); + return worker; + } + return new Worker(devUrl); +} diff --git a/src/oauth/generic-account-failover.ts b/src/oauth/generic-account-failover.ts index bfd86f8e97c..904791f6af2 100644 --- a/src/oauth/generic-account-failover.ts +++ b/src/oauth/generic-account-failover.ts @@ -41,7 +41,6 @@ import type { OcxConfig, OcxProviderConfig } from "../types"; export const GENERIC_OAUTH_MAX_FAILOVERS_PER_REQUEST = 3; const DEFAULT_COOLDOWN_MS = 60_000; -const MAX_COOLDOWN_MS = 15 * 60_000; /** * How long a presence answer may be reused before the store is consulted again. @@ -349,12 +348,15 @@ export function rotateGenericOAuthAccountOn429( // A single stored account has nowhere to go; rotating to itself would just replay the 429. if (!set || set.accounts.length < 2) return null; - const parsed = parseRetryAfterMs(retryAfterHeader, now, { preserveImmediate: true }); + // `preserveServerDelay` keeps the delay the server actually stated, bounded by the parser's + // one-day ceiling, exactly as the combo path does. Truncating it locally only guarantees a + // second 429 on an account we were told to leave alone. + const parsed = parseRetryAfterMs(retryAfterHeader, now, { preserveImmediate: true, preserveServerDelay: true }); // An account whose allowance is provably spent gets a reset-aligned cooldown instead of // the default minute: retrying it every 60s until the window rolls over is pure waste. // A Retry-After from upstream still wins — it is the server's own instruction. const exhausted = parsed === undefined ? exhaustedCooldownMs(providerName, failedAccountId, now) : null; - const cooldownMs = exhausted ?? Math.min(parsed ?? DEFAULT_COOLDOWN_MS, MAX_COOLDOWN_MS); + const cooldownMs = exhausted ?? parsed ?? DEFAULT_COOLDOWN_MS; const family = classifyModelFamilyForQuota(providerName, requestedModelId); health.set(healthKey(providerName, failedAccountId, family), { cooldownUntil: now + cooldownMs, diff --git a/src/oauth/index.ts b/src/oauth/index.ts index 8a2cd9d72ee..84d76a6d520 100644 --- a/src/oauth/index.ts +++ b/src/oauth/index.ts @@ -22,11 +22,12 @@ import { mergeAccountCredential, normalizeAuthStoreBuffer, readOAuthRefreshIntent, - removeAccount, + rollbackCredentialWriteIfMatch, saveAccountCredential, saveCredential, - setActiveAccount, + saveCredentialWithReceipt, writeOAuthRefreshIntent, + type OAuthCredentialWriteReceipt, type OAuthRefreshIntent, type OAuthRefreshIntentCleanupPending, } from "./store"; @@ -1590,33 +1591,15 @@ export function upsertOAuthProvider(config: OcxConfig, provider: string): void { interface RunLoginDeps { saveCredential?: typeof saveCredential; + saveCredentialWithReceipt?: typeof saveCredentialWithReceipt; saveAccountCredential?: typeof saveAccountCredential; loadConfig?: typeof loadConfig; saveConfig?: typeof saveConfig; settleKiroLoginTransaction?: typeof settleKiroLoginTransaction; - removeAccount?: typeof removeAccount; - setActiveAccount?: typeof setActiveAccount; + rollbackCredentialWrite?: typeof rollbackCredentialWriteIfMatch; assertCurrentOwner?: () => void; } -/** Roll back only accounts created by this forced login, preserving concurrent refreshes of others. */ -async function rollbackForcedKiroAccountWrite( - provider: string, - previousActiveId: string | undefined, - previousAccountIds: ReadonlySet, - deps: Pick, -): Promise { - const set = getAccountSet(provider); - if (!set) return; - for (const account of [...set.accounts]) { - if (previousAccountIds.has(account.id)) continue; - await (deps.removeAccount ?? removeAccount)(provider, account.id); - } - if (previousActiveId && getAccountCredential(provider, previousActiveId)) { - await (deps.setActiveAccount ?? setActiveAccount)(provider, previousActiveId); - } -} - /** Run the login flow, persist the credential + upsert the provider entry to disk, return cred. */ export async function runLogin( provider: string, @@ -1639,9 +1622,7 @@ export async function runLogin( // loginKiro keys its pending CLI-session transaction by object identity. Keep this exact object // for settlement even when source normalization below creates a derived credential object. const shouldRollbackKiroAccounts = provider === "kiro" && opts?.forceLogin === true; - const previousKiroAccounts = shouldRollbackKiroAccounts ? getAccountSet(provider) : undefined; - const previousKiroActiveId = previousKiroAccounts?.activeAccountId; - const previousKiroAccountIds = new Set(previousKiroAccounts?.accounts.map(account => account.id) ?? []); + let kiroCredentialWrite: OAuthCredentialWriteReceipt | null = null; const loginProviderConfig = preflightConfig ? (def.resolveProviderConfig?.(preflightConfig) ?? preflightConfig.providers[provider] ?? def.providerConfig) : def.providerConfig; @@ -1674,10 +1655,19 @@ export async function runLogin( assertBeforePersist: deps.assertCurrentOwner, }); } else { - await (deps.saveCredential ?? saveCredential)(provider, cred, { + const saveOptions = { preserveIdentityless: opts?.forceLogin === true, assertBeforePersist: deps.assertCurrentOwner, - }); + }; + if (shouldRollbackKiroAccounts && !deps.saveCredential) { + kiroCredentialWrite = await (deps.saveCredentialWithReceipt ?? saveCredentialWithReceipt)( + provider, + cred, + saveOptions, + ); + } else { + await (deps.saveCredential ?? saveCredential)(provider, cred, saveOptions); + } } if (provider !== "chatgpt") { // Re-run against post-credential state so same-provider API-key additions, removals, @@ -1695,9 +1685,9 @@ export async function runLogin( } } catch (error) { const errors: unknown[] = [error]; - if (shouldRollbackKiroAccounts) { + if (kiroCredentialWrite) { try { - await rollbackForcedKiroAccountWrite(provider, previousKiroActiveId, previousKiroAccountIds, deps); + await (deps.rollbackCredentialWrite ?? rollbackCredentialWriteIfMatch)(kiroCredentialWrite); } catch (rollbackError) { errors.push(rollbackError); } diff --git a/src/oauth/store.ts b/src/oauth/store.ts index a5b57ebcadf..e834b60ef11 100644 --- a/src/oauth/store.ts +++ b/src/oauth/store.ts @@ -765,7 +765,7 @@ function serializeMutation(work: () => Promise, retainedValues: readonly u drainOAuthMutations(); return result; } -export function mutateStore(fn:(store:AuthStore)=>T|Promise, retainedValues: readonly unknown[] = [], options?: { waitMs?: number; assertBeforePersist?: () => void; scrubLegacyBackup?: (result: T) => readonly string[] }):Promise{return serializeMutation(async()=>{const guard=await createOAuthFileLock({path:getAuthStoreLockPath(),staleAfterMs:30000}).acquire();try{ +export function mutateStore(fn:(store:AuthStore)=>T|Promise, retainedValues: readonly unknown[] = [], options?: { waitMs?: number; assertBeforePersist?: () => void; scrubLegacyBackup?: (result: T) => readonly string[]; finalizeResult?: (result: T, store: AuthStore) => void }):Promise{return serializeMutation(async()=>{const guard=await createOAuthFileLock({path:getAuthStoreLockPath(),staleAfterMs:30000}).acquire();try{ const { store, hadLegacy } = loadAuthStoreInternal(); if (hadLegacy) backupLegacyOnce(); const selections = new Map(Object.entries(store).map(([provider, set]) => [provider, { @@ -798,6 +798,9 @@ export function mutateStore(fn:(store:AuthStore)=>T|Promise, retainedValue changedProviders.push(provider); } } + // Receipt-producing mutations need the revision assigned by the bookkeeping above, not the + // provisional value visible inside their callback. Finalization cannot await or mutate disk. + options?.finalizeResult?.(result, store); persist(store); if (scrubbedProviders.length > 0) scrubLegacyBackup(scrubbedProviders); for (const provider of changedProviders) publishAccountSelection(provider, "oauth"); @@ -819,63 +822,132 @@ export function getCredential(provider: string): OAuthCredentials | null { * active slot / whole set instead. An explicit add-account login can preserve the legacy slot; * an identity-less credential then gets its deterministic refresh-derived account id. */ -export async function saveCredential( +export interface OAuthCredentialWriteReceipt { + provider: string; + accountId: string; + credentialGeneration: string; + selectionRevision: string | undefined; + previousActiveAccountId: string | undefined; + previousAccount: ProviderAccount | undefined; +} + +export async function saveCredentialWithReceipt( provider: string, cred: OAuthCredentials, opts: { preserveIdentityless?: boolean; assertBeforePersist?: () => void } = {}, -): Promise { +): Promise { const safe = normalizeCredential(cred); - if (!safe) return; - await mutateStore(store => { + if (!safe) return null; + return await mutateStore(store => { const set = store[provider]; + const previousActiveAccountId = set?.activeAccountId; + const previousAccounts = new Map( + set?.accounts.map(account => [account.id, structuredClone(account)]) ?? [], + ); // Login explicitly selects an account, including a re-login to the same slot. if (set) set.selectionRevision = randomUUID(); const identity = safe.accountId ?? safe.email; + let accountId: string; if (!set || SINGLE_SLOT_PROVIDERS.has(provider)) { const id = newAccountId(safe); store[provider] = { activeAccountId: id, accounts: [{ id, credential: safe, addedAt: Date.now() }] }; - return; - } - if (identity) { + accountId = id; + } else if (identity) { const existing = set.accounts.find(a => (a.credential.accountId ?? a.credential.email) === identity); if (existing) { existing.credential = safe; delete existing.needsReauth; set.activeAccountId = existing.id; - return; + accountId = existing.id; + } else { + // Legacy migration: a pre-identity row (no accountId/email) for this provider is the + // SAME human re-logging in after the identity extraction shipped — upgrading the + // active identity-less row in place prevents a stale duplicate that stays selectable + // and would re-refresh into a second row with the same identity. + const active = set.accounts.find(a => a.id === set.activeAccountId); + if (!opts.preserveIdentityless && active && active.credential.accountId === undefined && active.credential.email === undefined) { + active.credential = safe; + delete active.needsReauth; + accountId = active.id; + } else { + const id = distinctAccountId(safe, set.accounts); + set.accounts.push({ id, credential: safe, addedAt: Date.now() }); + set.activeAccountId = id; + accountId = id; + } } - // Legacy migration: a pre-identity row (no accountId/email) for this provider is the - // SAME human re-logging in after the identity extraction shipped — upgrading the - // active identity-less row in place prevents a stale duplicate that stays selectable - // and would re-refresh into a second row with the same identity. + } else if (opts.preserveIdentityless) { + const id = distinctAccountId(safe, set.accounts); + set.accounts.push({ id, credential: safe, addedAt: Date.now() }); + set.activeAccountId = id; + accountId = id; + } else { + // No identity during a normal login: replace the active slot in place. const active = set.accounts.find(a => a.id === set.activeAccountId); - if (!opts.preserveIdentityless && active && active.credential.accountId === undefined && active.credential.email === undefined) { + if (active) { active.credential = safe; delete active.needsReauth; - return; + accountId = active.id; + } else { + const id = distinctAccountId(safe, set.accounts); + set.accounts.push({ id, credential: safe, addedAt: Date.now() }); + set.activeAccountId = id; + accountId = id; } - const id = distinctAccountId(safe, set.accounts); - set.accounts.push({ id, credential: safe, addedAt: Date.now() }); - set.activeAccountId = id; - return; } - if (opts.preserveIdentityless) { - const id = distinctAccountId(safe, set.accounts); - set.accounts.push({ id, credential: safe, addedAt: Date.now() }); - set.activeAccountId = id; - return; + return { + provider, + accountId, + credentialGeneration: credentialGeneration(safe), + selectionRevision: store[provider]?.selectionRevision, + previousActiveAccountId, + previousAccount: previousAccounts.get(accountId), + }; + }, [provider, safe], { + assertBeforePersist: opts.assertBeforePersist, + finalizeResult: (receipt, store) => { + receipt.selectionRevision = store[provider]?.selectionRevision; + }, + }); +} + +/** Ordinary callers do not acquire rollback authority merely by saving a credential. */ +export async function saveCredential( + provider: string, + cred: OAuthCredentials, + opts: { preserveIdentityless?: boolean; assertBeforePersist?: () => void } = {}, +): Promise { + await saveCredentialWithReceipt(provider, cred, opts); +} + +/** Compensate one owned login write without deleting or selecting over concurrent work. */ +export async function rollbackCredentialWriteIfMatch( + receipt: OAuthCredentialWriteReceipt, +): Promise<"rolled-back" | "stale"> { + return await mutateStore(store => { + const set = store[receipt.provider]; + const index = set?.accounts.findIndex(account => account.id === receipt.accountId) ?? -1; + if (!set || index < 0) return "stale" as const; + const current = set.accounts[index]!; + if (credentialGeneration(current.credential) !== receipt.credentialGeneration) return "stale" as const; + // A later explicit selection of the same account owns that choice. Do not remove or rewrite + // the selected slot underneath it; the failed login can report failure without erasing newer state. + if (set.activeAccountId === receipt.accountId + && set.selectionRevision !== receipt.selectionRevision) return "stale" as const; + + if (receipt.previousAccount) set.accounts[index] = structuredClone(receipt.previousAccount); + else set.accounts.splice(index, 1); + if (set.accounts.length === 0) { + delete store[receipt.provider]; + return "rolled-back" as const; } - // No identity during a normal login: replace the active slot in place. - const active = set.accounts.find(a => a.id === set.activeAccountId); - if (active) { - active.credential = safe; - delete active.needsReauth; - } else { - const id = distinctAccountId(safe, set.accounts); - set.accounts.push({ id, credential: safe, addedAt: Date.now() }); - set.activeAccountId = id; + if (set.activeAccountId === receipt.accountId) { + const previousStillExists = receipt.previousActiveAccountId + && set.accounts.some(account => account.id === receipt.previousActiveAccountId); + set.activeAccountId = previousStillExists ? receipt.previousActiveAccountId! : set.accounts[0]!.id; } - }, [provider, safe], { assertBeforePersist: opts.assertBeforePersist }); + return "rolled-back" as const; + }, [receipt]); } /** diff --git a/src/remote-control/workspace-rpc.ts b/src/remote-control/workspace-rpc.ts index 15f8c7a3811..fff503d01d4 100644 --- a/src/remote-control/workspace-rpc.ts +++ b/src/remote-control/workspace-rpc.ts @@ -16,15 +16,30 @@ import { frameRemoteWorkspaceRpcMessage, } from "./workspace-rpc-framing"; -const REMOTE_WORKSPACE_RPC_VERSION = 1 as const; -const REMOTE_WORKSPACE_RPC_DEFAULT_TIMEOUT_MS = 30_000; +// Old endpoints execute request frames immediately and cannot safely participate in prepare/grant. +const REMOTE_WORKSPACE_RPC_VERSION = 2 as const; +const REMOTE_WORKSPACE_RPC_DEFAULT_TIMEOUT_MS = 65_000; +const REMOTE_WORKSPACE_RPC_MAX_TIMEOUT_MS = 120_000; const REMOTE_WORKSPACE_RPC_MAX_ACTIVE_REQUESTS = 8; interface RemoteWorkspaceRpcRequest { version: typeof REMOTE_WORKSPACE_RPC_VERSION; - kind: "request"; + kind: "prepare"; + timeoutMs: number; request: RemoteWorkspaceExecutionRequest; } +interface RemoteWorkspaceRpcCancel { + version: typeof REMOTE_WORKSPACE_RPC_VERSION; + kind: "cancel"; + requestId: string; +} + +interface RemoteWorkspaceRpcGrant { + version: typeof REMOTE_WORKSPACE_RPC_VERSION; + kind: "grant"; + requestId: string; +} + interface RemoteWorkspaceRpcResponse { version: typeof REMOTE_WORKSPACE_RPC_VERSION; kind: "response"; @@ -32,7 +47,7 @@ interface RemoteWorkspaceRpcResponse { result: RemoteWorkspaceToolResult; } -type RemoteWorkspaceRpcMessage = RemoteWorkspaceRpcRequest | RemoteWorkspaceRpcResponse; +type RemoteWorkspaceRpcMessage = RemoteWorkspaceRpcRequest | RemoteWorkspaceRpcResponse | RemoteWorkspaceRpcCancel | RemoteWorkspaceRpcGrant; interface PendingRequest { resolve(value: RemoteWorkspaceToolResult): void; @@ -101,8 +116,14 @@ function parseMessage(value: Uint8Array): RemoteWorkspaceRpcMessage { if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("invalid remote workspace RPC message"); const raw = parsed as Record; if (raw.version !== REMOTE_WORKSPACE_RPC_VERSION) throw new Error("unsupported remote workspace RPC version"); - if (raw.kind === "request") { - return { version: REMOTE_WORKSPACE_RPC_VERSION, kind: "request", request: parseRequest(raw.request) }; + if (raw.kind === "prepare" && Number.isSafeInteger(raw.timeoutMs) + && (raw.timeoutMs as number) >= 1 && (raw.timeoutMs as number) <= REMOTE_WORKSPACE_RPC_MAX_TIMEOUT_MS) { + return { + version: REMOTE_WORKSPACE_RPC_VERSION, + kind: "prepare", + timeoutMs: raw.timeoutMs as number, + request: parseRequest(raw.request), + }; } if (raw.kind === "response" && boundedIdentifier(raw.requestId)) { return { @@ -112,6 +133,9 @@ function parseMessage(value: Uint8Array): RemoteWorkspaceRpcMessage { result: parseResult(raw.result), }; } + if ((raw.kind === "cancel" || raw.kind === "grant") && boundedIdentifier(raw.requestId)) { + return { version: REMOTE_WORKSPACE_RPC_VERSION, kind: raw.kind, requestId: raw.requestId }; + } throw new Error("invalid remote workspace RPC message kind"); } @@ -132,7 +156,8 @@ export class EncryptedRemoteWorkspaceTransport implements RemoteWorkspaceTranspo constructor(private readonly options: EncryptedRemoteWorkspaceTransportOptions) { this.timeoutMs = options.timeoutMs ?? REMOTE_WORKSPACE_RPC_DEFAULT_TIMEOUT_MS; - if (!boundedIdentifier(options.executorDeviceId) || !Number.isSafeInteger(this.timeoutMs) || this.timeoutMs < 1) { + if (!boundedIdentifier(options.executorDeviceId) || !Number.isSafeInteger(this.timeoutMs) + || this.timeoutMs < 1 || this.timeoutMs > REMOTE_WORKSPACE_RPC_MAX_TIMEOUT_MS) { throw new Error("invalid encrypted remote workspace transport options"); } } @@ -150,22 +175,38 @@ export class EncryptedRemoteWorkspaceTransport implements RemoteWorkspaceTranspo const response = new Promise((resolve, reject) => { const timer = setTimeout(() => { this.pending.delete(request.requestId); - reject(new Error("remote workspace request timed out")); + void this.sendCancellation(request.requestId); + reject(new Error("remote workspace request timed out; executor cancellation was requested")); }, this.timeoutMs); this.pending.set(request.requestId, { resolve, reject, timer }); }); + const pending = this.pending.get(request.requestId)!; + // Observe the response immediately: transport backpressure must not defer timeout delivery + // or leave a rejected response promise unobserved while a write is still waiting to settle. + void this.prepareAndGrant(request, pending); + return await response; + } + + private async prepareAndGrant(request: RemoteWorkspaceExecutionRequest, pending: PendingRequest): Promise { try { await this.sendMessage(encodeMessage({ version: REMOTE_WORKSPACE_RPC_VERSION, - kind: "request", + kind: "prepare", + timeoutMs: this.timeoutMs, request, })); + // Check again inside the serialized send queue: another write can delay this grant after + // prepare has settled. A request that has already timed out must never receive a grant. + await this.sendMessage(encodeMessage({ + version: REMOTE_WORKSPACE_RPC_VERSION, + kind: "grant", + requestId: request.requestId, + }), () => this.pending.get(request.requestId) === pending); } catch { // A failed encrypted write consumes a directional counter. Continuing would make every // later frame undecryptable, so fail every pending operation instead of waiting for timeout. this.close("remote workspace send failed"); } - return await response; } receiveCiphertext(value: Uint8Array): void { @@ -193,8 +234,9 @@ export class EncryptedRemoteWorkspaceTransport implements RemoteWorkspaceTranspo this.pending.clear(); } - private sendMessage(message: Uint8Array): Promise { + private sendMessage(message: Uint8Array, shouldSend: () => boolean = () => true): Promise { const operation = this.sendTail.then(async () => { + if (!shouldSend()) return; if (!this.online) throw new Error("remote workspace transport is closed"); for (const frame of frameRemoteWorkspaceRpcMessage(message)) { await this.options.sendCiphertext(this.options.cipher.encrypt(frame)); @@ -203,6 +245,19 @@ export class EncryptedRemoteWorkspaceTransport implements RemoteWorkspaceTranspo this.sendTail = operation.catch(() => {}); return operation; } + + private async sendCancellation(requestId: string): Promise { + try { + await this.sendMessage(encodeMessage({ + version: REMOTE_WORKSPACE_RPC_VERSION, + kind: "cancel", + requestId, + })); + } catch { + // A failed encrypted write consumes the send counter; the session cannot safely continue. + this.close("remote workspace cancellation send failed"); + } + } } export interface EncryptedRemoteWorkspaceExecutorEndpointOptions { @@ -218,7 +273,12 @@ export interface EncryptedRemoteWorkspaceExecutorEndpointOptions { /** Executor-side endpoint. It accepts only authenticated, ordered E2EE session frames. */ export class EncryptedRemoteWorkspaceExecutorEndpoint { private closed = false; - private readonly active = new Map(); + private readonly active = new Map; + request: RemoteWorkspaceExecutionRequest; + started: boolean; + }>(); private readonly reassembler = new RemoteWorkspaceRpcReassembler(); private sendTail: Promise = Promise.resolve(); @@ -240,7 +300,26 @@ export class EncryptedRemoteWorkspaceExecutorEndpoint { const requestPlaintext = this.reassembler.accept(this.options.cipher.decrypt(value)); if (!requestPlaintext) return; const message = parseMessage(requestPlaintext); - if (message.kind !== "request") throw new Error("executor received a remote workspace response"); + if (message.kind === "cancel") { + const active = this.active.get(message.requestId); + if (active) { + active.controller.abort(); + if (!active.started) { + clearTimeout(active.timer); + this.active.delete(message.requestId); + } + } + return; + } + if (message.kind === "grant") { + const active = this.active.get(message.requestId); + if (!active || active.controller.signal.aborted) return; + if (active.started) throw new Error("duplicate remote workspace execution grant"); + active.started = true; + await this.executeGranted(active.request, active.controller, active.timer); + return; + } + if (message.kind !== "prepare") throw new Error("executor received a remote workspace response"); if (message.request.executorDeviceId !== this.options.executorDeviceId) { throw new Error("remote workspace encrypted request targeted another executor"); } @@ -255,12 +334,25 @@ export class EncryptedRemoteWorkspaceExecutorEndpoint { throw new Error("remote workspace executor request limit reached"); } const controller = new AbortController(); - this.active.set(message.request.requestId, controller); + const timer = setTimeout(() => { + controller.abort(); + const active = this.active.get(message.request.requestId); + if (active && !active.started) this.active.delete(message.request.requestId); + }, message.timeoutMs); + this.active.set(message.request.requestId, { controller, timer, request: message.request, started: false }); + } + + private async executeGranted( + request: RemoteWorkspaceExecutionRequest, + controller: AbortController, + timer: ReturnType, + ): Promise { let result: RemoteWorkspaceToolResult; try { - result = await this.options.executor.invoke(message.request, controller.signal); + result = await this.options.executor.invoke(request, controller.signal); } finally { - this.active.delete(message.request.requestId); + clearTimeout(timer); + this.active.delete(request.requestId); } if (this.closed) return; let responsePlaintext: Uint8Array; @@ -268,14 +360,14 @@ export class EncryptedRemoteWorkspaceExecutorEndpoint { responsePlaintext = encodeMessage({ version: REMOTE_WORKSPACE_RPC_VERSION, kind: "response", - requestId: message.request.requestId, + requestId: request.requestId, result, }); } catch { responsePlaintext = encodeMessage({ version: REMOTE_WORKSPACE_RPC_VERSION, kind: "response", - requestId: message.request.requestId, + requestId: request.requestId, result: { ok: false, error: "remote workspace result exceeded the encrypted frame limit" }, }); } @@ -286,7 +378,10 @@ export class EncryptedRemoteWorkspaceExecutorEndpoint { if (this.closed) return; this.closed = true; this.reassembler.clear(); - for (const controller of this.active.values()) controller.abort(); + for (const active of this.active.values()) { + clearTimeout(active.timer); + active.controller.abort(); + } this.active.clear(); this.options.cipher.destroy(); } diff --git a/src/router.ts b/src/router.ts index 97fad209f8e..b7a4032792c 100644 --- a/src/router.ts +++ b/src/router.ts @@ -627,6 +627,7 @@ function routeModelInternal( bypassCombos: boolean, policyEvidence?: PolicyRequestEvidence, allowCompactionNativeFallback = false, + preview = false, ): RouteResult { const slash = modelId.indexOf("/"); // Policy namespace is system-reserved: an explicit `policy/` or a @@ -694,7 +695,7 @@ function routeModelInternal( } if (!bypassCombos && !preservesPhysicalComboProvider(config)) { - const combo = tryPickComboModel(config, modelId); + const combo = tryPickComboModel(config, modelId, preview); if (combo) { const concrete = `${combo.target.provider}/${combo.target.model}`; // The selected target is already a concrete provider/model reference. Resolve it without @@ -894,6 +895,11 @@ export function routeModel( return routeWithDecisionTrace(config, modelId, route); } +/** Resolve a route for capability inspection without creating combo selection state. */ +export function previewRouteModel(config: OcxConfig, modelId: string): RouteResult { + return routeWithDecisionTrace(config, modelId, routeModelInternal(config, modelId, false, undefined, false, true)); +} + /** * Route a client-selected compaction model. Codex may send a bare native model * even when its ordinary turns are configured for another provider; in that diff --git a/src/server/chat-completions.ts b/src/server/chat-completions.ts index 4ce9efc985c..39a15eaec44 100644 --- a/src/server/chat-completions.ts +++ b/src/server/chat-completions.ts @@ -341,7 +341,8 @@ async function handleChatCompletionsWithBudget( } else if (internalBody.store === undefined) { internalBody.store = false; } - if (settledRoute && internalBody.reasoning !== undefined) { + if (settledRoute && !settledRoute.combo && settledRoute.routeKind !== "policy" + && internalBody.reasoning !== undefined) { const { stripEmptyLadderEffort, supportedLadderFor } = await import("./effort-policy"); const ladder = supportedLadderFor({ provider: settledRoute.provider, modelId: settledRoute.modelId }); const next = stripEmptyLadderEffort(internalBody.reasoning, ladder); diff --git a/src/server/responses/core-normalize.ts b/src/server/responses/core-normalize.ts index e228eac0c43..86b0ae546ed 100644 --- a/src/server/responses/core-normalize.ts +++ b/src/server/responses/core-normalize.ts @@ -12,7 +12,7 @@ import { subagentFallbackNeedsModelEntitlements } from "../../codex/subagent-mod import { MAIN_CODEX_ACCOUNT_ID } from "../../codex/main-account"; import type { RequestLogContext } from "../request-log"; import type { HandleResponsesOptions } from "./core-options"; -import { prepareEffortNormalization } from "../effort-policy"; +import { prepareEffortNormalization, stripEmptyLadderEffort, supportedLadderFor } from "../effort-policy"; import { resolveOpenCodeGoTransport } from "../../providers/opencode-go-transport"; import { getOrAllocateRequestSessionLane } from "../request-log-conversation"; import { shouldPreparePlaintextV2AgentMessages } from "../../responses/plaintext-v2-agent-messages"; @@ -307,6 +307,18 @@ export async function applyFinalRouteRequestNormalization(args: { logCtx.requestedEffort = `${logCtx.requestedEffort ?? "max"}->${clamped}`; } } + // Chat ingress cannot strip effort against a provisional policy pick. Apply the + // concrete target's restriction to BOTH adapter options and the raw wire copy; + // policy-fallback retains the original body before this attempt-local mutation. + if (inboundWire === "chat" && supportedLadderFor(route)?.length === 0) { + parsed.options.reasoning = undefined; + const raw = parsed._rawBody as { reasoning?: unknown } | undefined; + if (raw) { + const reasoning = stripEmptyLadderEffort(raw.reasoning, []); + if (reasoning === undefined) delete raw.reasoning; + else raw.reasoning = reasoning; + } + } recordAttemptRequestedEffort(logCtx); logCtx.modelSupportsServiceTier = SERVICE_TIER_ADAPTERS.has(route.provider.adapter) ? modelServiceTierSupport diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index 75e97f44d9c..24ffc9f126c 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -160,7 +160,12 @@ export async function executeResponsesSidecars( retryAfter: string | null, responseHeaders?: Headers, retryParsed?: OcxParsedRequest, - ): Promise => { + ): Promise<{ adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null> => { + // Which credential axis actually moved. The main routed path already reports these three + // separately (`adapter-dispatch`: key-429 / anthropic-oauth-429 / oauth-account-429); the + // sidecar loops used to flatten all three to `key-429`, so an account rotation read as a key + // rotation in the attempt row and in the Logs UI. + let recoveryKind: AttemptRecoveryKind = "key-429"; const rotated = rotateProviderTransportOn429(config, route.providerName, route.provider, { retryAfter, now: Date.now(), @@ -209,6 +214,7 @@ export async function executeResponsesSidecars( hop.permit?.release(); return null; } + recoveryKind = "oauth-account-429"; hop.permit?.use(); } else if ( // Anthropic's pool is excluded from generic failover, so without this arm a 429 inside a @@ -252,6 +258,7 @@ export async function executeResponsesSidecars( hop.permit?.release(); return null; } + recoveryKind = "anthropic-oauth-429"; hop.permit?.use(); } else { // No key pool, no generic OAuth roster, no Anthropic pool could produce a replacement @@ -281,7 +288,7 @@ export async function executeResponsesSidecars( provider: route.provider, adapterName: rotatedAdapter.name, }); - return rotatedAdapter; + return { adapter: rotatedAdapter, recoveryKind }; }; if ((imgPlan || vidPlan) && wsPlan) { // Web search takes priority when both are active — the media bridge cannot run diff --git a/src/server/search.ts b/src/server/search.ts index 51e418346e9..60db8c25ab3 100644 --- a/src/server/search.ts +++ b/src/server/search.ts @@ -38,8 +38,8 @@ import { resolveFirstUsableOpenAiSidecar, type ExactOpenAiSidecarAccount, } from "../providers/openai-sidecar"; -import { routeModel } from "../router"; -import { handleAlphaSearchSidecarFallback } from "../web-search/alpha-search"; +import { previewRouteModel, routeModel } from "../router"; +import { handleAlphaSearchSidecarFallback, handleDevinAlphaSearch } from "../web-search/alpha-search"; import { readJsonRequestBody, resolveInboundBodyLimitBytes } from "./request-decompress"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "./auth-cors"; import type { RequestLogContext } from "./request-log"; @@ -65,11 +65,6 @@ export async function handleSearch( turnAdmissionLease?: AdmissionLease, admission?: DataPlaneAdmission, ): Promise { - try { validateForwardAdmissionCredential(req.headers, config); } - catch (err) { - if (err instanceof ForwardAdmissionCredentialError) return formatErrorResponse(401, "authentication_error", err.message); - throw err; - } let body: unknown; try { body = await readJsonRequestBody(req, undefined, resolveInboundBodyLimitBytes(config.maxInboundBodyBytes)); @@ -79,6 +74,32 @@ export async function handleSearch( const model = (body as { model?: unknown } | null)?.model; if (typeof model === "string" && model) logCtx.model = model; + if (typeof model === "string" && model.trim()) { + try { + const route = previewRouteModel(config, model); + if (route.providerName === "devin" || route.staticPolicy.model.adapter === "devin") { + const denial = admissionScopeDenial(config, admission, model, route); + if (denial) return denial; + logCtx.provider = route.providerName; + logCtx.routeDecision = route.routeDecision; + return handleDevinAlphaSearch( + body, + "devin", + config.search?.timeoutMs ?? SEARCH_UPSTREAM_TIMEOUT_MS, + req.signal, + ); + } + } catch { + // Preview is advisory: existing relay/fallback still owns unsupported or unroutable models. + } + } + + try { validateForwardAdmissionCredential(req.headers, config); } + catch (err) { + if (err instanceof ForwardAdmissionCredentialError) return formatErrorResponse(401, "authentication_error", err.message); + throw err; + } + let exactAccount: ExactOpenAiSidecarAccount | undefined; let relayBody = body; const accountNamespace = typeof model === "string" diff --git a/src/storage/policy-job.ts b/src/storage/policy-job.ts index 50dc5cbb7b5..dd03a9e76a1 100644 --- a/src/storage/policy-job.ts +++ b/src/storage/policy-job.ts @@ -6,6 +6,7 @@ * proxy event loop stays responsive. */ import type { CleanupMode, CleanupResult } from "./cleanup"; +import { spawnWorker } from "../lib/worker-embed"; import { resolveCodexHomeDir } from "../codex/home"; import { tryBeginStorageMutation, @@ -310,7 +311,7 @@ function runInWorker(opts: RequestPolicyRunOptions & { blockMs?: number }): Prom let settled = false; let worker: Worker; try { - worker = new Worker(new URL("./policy-worker.ts", import.meta.url).href); + worker = spawnWorker(new URL("./policy-worker.ts", import.meta.url).href, "policy-worker"); reservation.bind(worker); } catch (error) { reservation.release(); diff --git a/src/storage/restore-job.ts b/src/storage/restore-job.ts index f338a32b02d..3d2e730ca21 100644 --- a/src/storage/restore-job.ts +++ b/src/storage/restore-job.ts @@ -9,6 +9,7 @@ * `storage_mutation_busy` (409) instead of queueing. */ import { resolveCodexHomeDir } from "../codex/home"; +import { spawnWorker } from "../lib/worker-embed"; import { restoreTrashEntry, type RestoreResult, type RestoreTestHooks } from "./cleanup"; import { resetStorageMutationCoordinatorForTests, @@ -167,7 +168,7 @@ function runInWorker(opts: { let settled = false; let worker: Worker; try { - worker = new Worker(new URL("./restore-worker.ts", import.meta.url).href); + worker = spawnWorker(new URL("./restore-worker.ts", import.meta.url).href, "restore-worker"); reservation.bind(worker); } catch (error) { reservation.release(); diff --git a/src/update/transactional-install.mjs b/src/update/transactional-install.mjs index 331b19b5180..44dd47e4166 100644 --- a/src/update/transactional-install.mjs +++ b/src/update/transactional-install.mjs @@ -189,8 +189,12 @@ function createOwnedStage(scopeDir, pkgName, deps = {}) { pid: process.pid, createdAt: (deps.now ?? Date.now)(), }), { flag: "wx" }); + // npm's strict script policy plans the global tree before it creates the prefix layout, so + // `-g --prefix` into a bare stage fails with ENOENT on /lib (#5760). POSIX global + // prefixes keep packages under lib/; Windows installs into the prefix itself. + if (process.platform !== "win32") mkdir(join(stageRoot, "lib")); } catch (error) { - // Still empty and created by this call: remove it rather than leave an unmarked stage. + // Created by this call and not yet handed to npm: remove it rather than leave a partial stage. try { rmSync(stageRoot, { recursive: true, force: true }); } catch { /* reported by the caller */ } throw error; } diff --git a/src/web-search/alpha-search.ts b/src/web-search/alpha-search.ts index 1b4489bfbe5..cd3d9b98581 100644 --- a/src/web-search/alpha-search.ts +++ b/src/web-search/alpha-search.ts @@ -12,12 +12,14 @@ * and pulling it in recreates the cycle sidecar-providers.ts exists to avoid. */ import { formatErrorResponse } from "../bridge"; +import { signalWithTimeout } from "../lib/abort"; import { redactSecretString } from "../lib/redact"; import { sidecarEnter } from "../lib/sidecar-tracker"; import { admissionScopeDenial } from "../server/admission-model-scope"; import type { DataPlaneAdmission } from "../server/auth-cors"; import type { OcxConfig, OcxProviderConfig, OcxWebSearchSidecarConfig } from "../types"; import { runAnthropicWebSearch } from "./anthropic-executor"; +import { resolveDevinWebSearchSnapshot, runDevinWebSearch } from "./devin-executor"; import { runExaWebSearch } from "./exa-executor"; import type { SidecarOutcome, SidecarSettings } from "./executor"; import { runGeminiWebSearch } from "./gemini-executor"; @@ -203,6 +205,75 @@ async function runAlphaSearchQuery( } } +function raceAbort(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + promise.then( + value => { signal.removeEventListener("abort", onAbort); resolve(value); }, + error => { signal.removeEventListener("abort", onAbort); reject(error); }, + ); + }); +} + +export async function handleDevinAlphaSearch( + body: unknown, + providerName: string, + timeoutMs: number, + signal?: AbortSignal, +): Promise { + const queries = extractAlphaSearchQueries(body); + if (queries.length === 0) { + return formatErrorResponse( + 400, + "invalid_request_error", + "Built-in web search request is missing a usable query (commands.search_query, query, q, or search_query).", + ); + } + const deadline = signalWithTimeout(timeoutMs, signal); + try { + const resolved = await raceAbort(resolveDevinWebSearchSnapshot(providerName), deadline.signal); + if ("error" in resolved) { + return formatErrorResponse(502, "upstream_error", `devin web search failed: ${redactSecretString(resolved.error)}`); + } + const texts: string[] = []; + const sources: SidecarOutcome["sources"] = []; + for (const query of queries) { + const outcome = await runDevinWebSearch(query, resolved.snapshot, deadline.signal); + if (outcome.error) { + if (signal?.aborted) { + return formatErrorResponse(499, "client_closed_request", "search request canceled by client"); + } + if (deadline.signal.aborted) { + return formatErrorResponse(504, "upstream_error", "devin web search timed out"); + } + const detail = redactSecretString(outcome.error); + return formatErrorResponse(502, "upstream_error", `devin web search failed: ${detail}`); + } + texts.push(queries.length > 1 ? `Results for "${query}":\n${outcome.text}` : outcome.text); + for (const source of outcome.sources) { + if (!sources.some(existing => existing.url === source.url)) sources.push(source); + } + } + return new Response(JSON.stringify(formatAlphaSearchBody(texts.join("\n\n"), sources)), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } catch (error) { + if (signal?.aborted) { + return formatErrorResponse(499, "client_closed_request", "search request canceled by client"); + } + if (deadline.signal.aborted) { + return formatErrorResponse(504, "upstream_error", "devin web search timed out"); + } + const detail = redactSecretString(error instanceof Error ? error.message : String(error)); + return formatErrorResponse(502, "upstream_error", `devin web search failed: ${detail}`); + } finally { + deadline.cleanup(); + } +} + function formatAlphaSearchBody(text: string, sources: SidecarOutcome["sources"]): { encrypted_output: null; output: string; diff --git a/src/web-search/devin-executor.ts b/src/web-search/devin-executor.ts new file mode 100644 index 00000000000..dde633c89f5 --- /dev/null +++ b/src/web-search/devin-executor.ts @@ -0,0 +1,192 @@ +import { buildMetadata } from "../adapters/devin/cloud-direct/metadata"; +import { encodeMessage, encodeString, encodeVarintField, iterFields } from "../adapters/devin/cloud-direct/wire"; +import { cancelBodyOnAbort } from "../lib/abort"; +import { readBoundedResponseBytes } from "../lib/bounded-body"; +import { redactSecretString } from "../lib/redact"; +import { sidecarEnter } from "../lib/sidecar-tracker"; +import { getValidAccessTokenSnapshot, publicOAuthAuthenticationErrorMessage, type OAuthAccessSnapshot } from "../oauth"; +import { captureOAuthAccountSelection, commitOAuthAccountSelection } from "../oauth/store"; +import { resolveDevinApiBaseUrl } from "../oauth/devin/api-base"; +import type { SidecarOutcome } from "./executor"; +import { MAX_SIDECAR_RESPONSE_BYTES, type WebSearchSource } from "./parse"; +import { safeWebSearchSources } from "./sources"; + +const DEVIN_WEB_SEARCH_PATH = "/exa.api_server_pb.ApiServerService/GetWebSearchResults"; +const DEVIN_WEB_SEARCH_RESULTS = 5; +const DEVIN_WEB_SEARCH_SNIPPET_CHARS = 2_000; + +interface DevinSearchResult extends WebSearchSource { + snippet?: string; +} + +function assertCompleteProtobuf(buf: Buffer): void { + let offset = 0; + while (offset < buf.length) { + const readVarint = (): bigint => { + let value = 0n; + for (let shift = 0n; shift < 70n; shift += 7n) { + if (offset >= buf.length) throw new Error("truncated varint"); + const byte = buf[offset++]!; + value |= BigInt(byte & 0x7f) << shift; + if ((byte & 0x80) === 0) return value; + } + throw new Error("overlong varint"); + }; + const tag = readVarint(); + const wire = Number(tag & 7n); + if (tag >> 3n === 0n) throw new Error("invalid field number"); + if (wire === 0) readVarint(); + else if (wire === 1) offset += 8; + else if (wire === 2) { + const length = Number(readVarint()); + if (!Number.isSafeInteger(length)) throw new Error("invalid field length"); + offset += length; + } else if (wire === 5) offset += 4; + else throw new Error("unsupported wire type"); + if (offset > buf.length) throw new Error("truncated field"); + } +} + +function fields(buf: Buffer): ReturnType { + assertCompleteProtobuf(buf); + return iterFields(buf); +} + +function decodeText(value: Buffer): string | undefined { + try { + return new TextDecoder("utf-8", { fatal: true }).decode(value); + } catch { + return undefined; + } +} + +function chunkText(chunk: Buffer): string | undefined { + for (const field of fields(chunk)) { + if (field.num === 1 && field.wire === 2 && Buffer.isBuffer(field.value)) return decodeText(field.value); + if (field.num === 3 && field.wire === 2 && Buffer.isBuffer(field.value)) { + for (const markdownField of fields(field.value)) { + if (markdownField.num === 2 && markdownField.wire === 2 && Buffer.isBuffer(markdownField.value)) { + return decodeText(markdownField.value); + } + } + } + } + return undefined; +} + +function parseResult(buf: Buffer): DevinSearchResult | undefined { + let url: string | undefined; + let title: string | undefined; + let text: string | undefined; + let summary: string | undefined; + const chunks: string[] = []; + for (const field of fields(buf)) { + if (field.wire !== 2 || !Buffer.isBuffer(field.value)) continue; + if (field.num === 2) text = decodeText(field.value); + else if (field.num === 3) url = decodeText(field.value); + else if (field.num === 4) title = decodeText(field.value); + else if (field.num === 6) { + const chunk = chunkText(field.value); + if (chunk) chunks.push(chunk); + } else if (field.num === 7) summary = decodeText(field.value); + } + if (!url) return undefined; + const snippet = (summary || text || chunks.join("\n")).trim().slice(0, DEVIN_WEB_SEARCH_SNIPPET_CHARS); + return { url, ...(title ? { title } : {}), ...(snippet ? { snippet } : {}) }; +} + +/** Map Cognition's GetWebSearchResults protobuf response to the shared search outcome. */ +export function mapDevinWebSearchResponse(buf: Buffer): SidecarOutcome { + const parsed: DevinSearchResult[] = []; + let summary = ""; + for (const field of fields(buf)) { + if (field.wire !== 2 || !Buffer.isBuffer(field.value)) continue; + if (field.num === 1) { + const result = parseResult(field.value); + if (result) parsed.push(result); + } else if (field.num === 3) { + summary = decodeText(field.value)?.trim().slice(0, DEVIN_WEB_SEARCH_SNIPPET_CHARS) ?? ""; + } + } + const sources = safeWebSearchSources(parsed); + if (sources.length === 0) return { text: "", sources: [], error: "devin web search returned no results" }; + const byUrl = new Map(parsed.map(result => [result.url, result])); + const lines = sources.map(source => { + const result = byUrl.get(source.url); + return `- ${source.title ?? source.url}: ${result?.snippet || "(no excerpt)"} [${source.url}]`; + }); + return { text: [summary, `Search results:\n${lines.join("\n")}`].filter(Boolean).join("\n\n"), sources }; +} + +/** Resolve one account snapshot for an entire alpha/search request. */ +export async function resolveDevinWebSearchSnapshot( + providerName: string, +): Promise<{ snapshot: OAuthAccessSnapshot } | { error: string }> { + const credentialProvider = providerName === "devin-cli" ? "devin" : providerName; + try { + const selection = captureOAuthAccountSelection(credentialProvider); + if (!selection) return { error: "devin web search auth failed: no signed-in account" }; + const snapshot = await getValidAccessTokenSnapshot(credentialProvider); + const committed = await commitOAuthAccountSelection(credentialProvider, snapshot.accountId, { + expectedSelection: selection, + expectedCredentialGeneration: snapshot.generation, + requireUsableAccount: true, + }); + return committed ? { snapshot } : { error: "devin web search auth changed; retry the request" }; + } catch (error) { + return { error: `devin web search auth failed: ${publicOAuthAuthenticationErrorMessage(error)}` }; + } +} + +/** Execute one search through the signed-in Devin/Cognition account. */ +export async function runDevinWebSearch( + query: string, + snapshot: OAuthAccessSnapshot, + abortSignal: AbortSignal, +): Promise { + const apiServerUrl = resolveDevinApiBaseUrl(snapshot.apiBaseUrl); + const metadata = buildMetadata({ + apiKey: snapshot.accessToken, + sessionId: crypto.randomUUID(), + requestId: BigInt(Date.now()), + triggerId: crypto.randomUUID(), + }); + const body = Buffer.concat([ + encodeMessage(1, metadata), + encodeString(2, query), + encodeVarintField(3, DEVIN_WEB_SEARCH_RESULTS), + ]); + const sidecarExit = sidecarEnter("web-search"); + try { + const response = await fetch(`${apiServerUrl}${DEVIN_WEB_SEARCH_PATH}`, { + method: "POST", + headers: { "Content-Type": "application/proto", "Connect-Protocol-Version": "1" }, + body: new Uint8Array(body), + signal: abortSignal, + redirect: "error", + }); + const detachBodyGuard = cancelBodyOnAbort(response.body, abortSignal); + try { + const bounded = await readBoundedResponseBytes(response, { + maxBytes: MAX_SIDECAR_RESPONSE_BYTES, + signal: abortSignal, + }); + if (bounded.oversized) return { text: "", sources: [], error: "devin web search response exceeded byte bound" }; + if (!response.ok) return { text: "", sources: [], error: `devin web search HTTP ${response.status}` }; + try { + return mapDevinWebSearchResponse(Buffer.from(bounded.bytes)); + } catch { + return { text: "", sources: [], error: "devin web search returned malformed protobuf" }; + } + } finally { + detachBodyGuard(); + } + } catch (error) { + const kind = abortSignal.reason instanceof Error && abortSignal.reason.name === "TimeoutError" + ? "timeout" + : "connect_error"; + return { text: "", sources: [], error: `devin web search ${kind}: ${redactSecretString(error instanceof Error ? error.message : String(error))}` }; + } finally { + sidecarExit(); + } +} diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 76f5748130a..0f3784cb8ea 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -337,12 +337,19 @@ export interface WebSearchLoopDeps { * `retryParsed` is the exact iteration-local request the retry will be built from. The loop * sends a shallow copy of the outer parsed request, so a rotation that rebinds only the outer * object never reaches the wire. Optional so existing callers keep compiling. + * + * A rotation may cross ACCOUNTS, not just keys, and the attempt row is where an operator reads + * which one happened. A rotator that knows which kind it performed returns it alongside the + * adapter -- the same `{ adapter, recoveryKind }` shape `onCredentialError` already uses below. */ on429?: ( retryAfterHeader: string | null, responseHeaders?: Headers, retryParsed?: OcxParsedRequest, - ) => ProviderAdapter | null | Promise; + ) => + | { adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } + | null + | Promise<{ adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null>; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; /** Called only when the final bridged Responses stream reaches completed or incomplete. */ @@ -568,10 +575,10 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise {}); } catch { /* already closed */ } - adapter = rotated; + adapter = rotated.adapter; // Stall-watchdog seam between bounded retry fetches (audit 011 B3). yield { type: "heartbeat" }; - prepared = await fetchOnce(adapter, "key-429"); + prepared = await fetchOnce(adapter, rotated.recoveryKind); } // Final headers have arrived. Clear only the deadline timer before ANY body read. diff --git a/structure/INDEX.md b/structure/INDEX.md index 79cff1af5d1..4f653bf3fb0 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -111,10 +111,10 @@ A source area can be described by more than one doc, because these docs are orga | `src/adapters/` | [`runtime.md`](runtime.md)
[`transports/byte-accounting.md`](transports/byte-accounting.md)
[`transports/responses-wire-shapes.md`](transports/responses-wire-shapes.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/cursor.md`](providers/cursor.md)
[`providers/chat-compat.md`](providers/chat-compat.md)
[`adapters/registry.md`](adapters/registry.md) | | `src/bridge.ts` | [`transports/responses.md`](transports/responses.md) | | `src/bridge/` | [`transports/responses.md`](transports/responses.md)
[`transports/responses-wire-shapes.md`](transports/responses-wire-shapes.md) | -| `src/chat/` | [`runtime.md`](runtime.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md)
[`providers-and-adapters.md`](providers-and-adapters.md) | +| `src/chat/` | [`runtime.md`](runtime.md)
[`transports/byte-accounting.md`](transports/byte-accounting.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/chat-compat.md`](providers/chat-compat.md) | | `src/claude/` | [`runtime.md`](runtime.md)
[`clients/claude-desktop.md`](clients/claude-desktop.md) | | `src/cli.ts` | [`runtime.md`](runtime.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | -| `src/cli/` | [`runtime.md`](runtime.md)
[`config.md`](config.md)
[`clients/claude-desktop.md`](clients/claude-desktop.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | +| `src/cli/` | [`runtime.md`](runtime.md)
[`config.md`](config.md)
[`clients/integrations.md`](clients/integrations.md)
[`clients/claude-desktop.md`](clients/claude-desktop.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | | `src/client/` | [`runtime.md`](runtime.md)
[`clients/claude-desktop.md`](clients/claude-desktop.md) | | `src/clients/` | [`clients/integrations.md`](clients/integrations.md) | | `src/codex/` | [`runtime.md`](runtime.md)
[`config.md`](config.md)
[`codex-home.md`](codex-home.md)
[`catalog.md`](catalog.md)
[`subagents.md`](subagents.md)
[`transports/responses-failover.md`](transports/responses-failover.md)
[`providers/openai-tiers.md`](providers/openai-tiers.md)
[`providers/openai-accounts.md`](providers/openai-accounts.md)
[`gui-and-management-api.md`](gui-and-management-api.md)
[`dashboard-and-usage.md`](dashboard-and-usage.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | diff --git a/structure/adapters/compatibility-lab.md b/structure/adapters/compatibility-lab.md index dcbb6da5400..72d374743fd 100644 --- a/structure/adapters/compatibility-lab.md +++ b/structure/adapters/compatibility-lab.md @@ -132,6 +132,32 @@ Live projection preserves the frozen `RouteSubjectV1` schema. Claim-gated scenar The two machine-readable Live V1 authority copies are required to be byte-identical. Runtime loading fails closed on byte drift before parsing. Scenario limits use `perArtifactBytes` as the single per-artifact execution-limit key; the artifact policy retains its independent per-artifact policy ceiling. +## CL-07 producer supervision + +An isolated fabric producer child is supervised through process exit, not through +its protocol stream: a parsed `result` line is stored, never settled, so an +executor cannot end its supervision early and keep mutating its scratch tree. +Protocol `error` lines, stream failures, and expired budgets latch a kill reason, +SIGKILL the child, and settle only at the run's decision point — so scratch +cleanup can never race a live producer. `exit` is the authoritative end of the +budget window: an already-met deadline still applies, otherwise both budget +timers are disarmed, and protocol bytes drained afterwards are judged at the +exit timestamp. A stored result is accepted only on a clean `code 0` exit +observed at `close`; a nonzero or signaled exit is a harness failure, and a +latched failure always wins settlement. `close` also waits for the child's +stdio, so after `exit` a bounded drain (`EXIT_DRAIN_MS`) lets in-flight protocol +data arrive; if `close` never follows, the run is rejected as an inconclusive +harness failure — a held-open pipe may mean a descendant escaped supervision or +simply that drainage stalled, so the result cannot be trusted and its scratch +cannot be cleaned while reporting success under a possibly-live process. +Rejections that could not observe `close` — a kill that produced neither `exit` +nor `close`, and any `exit` whose `close` never arrived — carry the deferred- +cleanup contract of an unconfirmed kill: the executor retains scratch and emits a fixed +manual-review warning without writing into producer-controlled paths. Later task creation +never sweeps these trees. Marker age and inherited-pipe closure are not termination leases. +After independently confirming all producer/descendant processes stopped, the operator may +review and remove the exact retained tree; parent exit does not grant automatic cleanup. + ## Scope guard CL-03 does not expose a management CLI/API or UI. Those surfaces remain CL-04+ work. Production request routing must not synchronously trigger Compatibility Lab probing or rebuild Lab evidence. diff --git a/structure/clients/integrations.md b/structure/clients/integrations.md index 4ed33f6013c..a15b853da50 100644 --- a/structure/clients/integrations.md +++ b/structure/clients/integrations.md @@ -8,6 +8,16 @@ promise is reversibility: apply snapshots first, writes atomically, records exac and refuses refresh, disable, or restore when the current file cannot be classified safely. Managed client targets are inspected without following a final symbolic link, and their atomic replacement addresses the named directory entry rather than resolving that link again at commit. +Uninstall runs the same coordinated disable path for every strict ownership record before removing +OpenCodex state, including the legacy Aside owner and every child profile store under +`integrations/aside-profiles//`. `src/cli/uninstall-integrations.ts` validates all stores +and registered Aside paths before mutation; `src/integrations/aside-profile-context.ts` supplies +the guarded child stores. An unreadable record, conflict, or failed compensation aborts config +removal and retains remaining recovery state. Earlier successful disables are not rolled back; +failed compensation can leave an intermediate client file. Inspect the reported client files and +retained snapshots before retrying; preserved recovery state does not prove restoration completed. + +> Decision record: [ADR-0107](../decisions/ADR-0107-uninstall-integration-recovery.md) Shared response support has a separate [bounded ingestion contract](../transports/inventory.md#bounded-response-ingestion-and-orcarouter-login): raw-byte callers own their byte and deadline budgets and inherit best-effort cancellation. diff --git a/structure/data-planes/search.md b/structure/data-planes/search.md index 81953f4ef83..3c9fd3fcb71 100644 --- a/structure/data-planes/search.md +++ b/structure/data-planes/search.md @@ -5,7 +5,18 @@ The opt-in key-auth Responses hosted-search bridge follows the ## Serving the relay without ChatGPT auth -`POST /v1/alpha/search` relays verbatim through a configured ChatGPT forward provider. When no +`POST /v1/alpha/search` first resolves a non-empty body `model` with `previewRouteModel()`. A route whose +effective adapter is `devin` uses `src/web-search/devin-executor.ts` and the active Devin OAuth +snapshot to call Cognition's non-inference `GetWebSearchResults` RPC. The account snapshot supplies +both the credential and its allowlisted tenant URL; the request rejects redirects and bounds timeout +and response bytes. This decision uses the resolved route, not a model-name family, so current and +future Devin-hosted model ids share the path without a model table. A round-robin preview copies +its selection state instead of claiming or advancing the turn's sticky slot. The RPC receives the +search query but no model id, and its decoded results return in the normal +`{ encrypted_output: null, output, results }` envelope. A configured key's destination scope is +checked against that resolved provider/model before dispatch. + +Other requests relay verbatim through a configured ChatGPT forward provider. When no forward candidate exists, an explicitly configured `webSearchSidecar.backend` of `anthropic`, `xai`, `gemini`, or `exa` serves the request instead, spending only that backend's own credential: `src/web-search/alpha-search.ts` runs the query through that backend's executor and diff --git a/structure/decisions/ADR-0107-uninstall-integration-recovery.md b/structure/decisions/ADR-0107-uninstall-integration-recovery.md new file mode 100644 index 00000000000..603687314e7 --- /dev/null +++ b/structure/decisions/ADR-0107-uninstall-integration-recovery.md @@ -0,0 +1,12 @@ +# ADR-0107 — decision recorded under "Client Integrations" + +- Contract owner: [clients/integrations.md](../clients/integrations.md) + +## Decision record + +- 목적과 의도: Restore every recorded third-party client contribution before uninstall deletes the ownership and snapshot evidence needed to do so safely. +- 기존 구현 및 제약 조건: Generic integration records lived under the OpenCodex config root and were removed by the ownership manifest, while uninstall restored native Codex, Grok, and Desktop state only. The ordinary disable writer already owns drift detection, snapshots, compensation, and per-client locking. +- 검토한 주요 대안: Leave external files unchanged; teach the config remover about third-party formats; restore snapshots wholesale; or invoke the existing coordinated disable workflow before config removal. +- 선택한 방식: Under the final client-lifecycle lease, strictly read root and Aside child-profile ownership stores, validate client IDs and registered profile paths before mutation, load one export roster, disable each recorded integration in deterministic order, verify its record retired, and abort config removal on any refusal or error. Enumerate persisted child stores rather than just desired/current profiles so orphaned ownership cannot be silently discarded. +- 다른 대안 대신 이 방식을 선택한 이유: Reusing the writer preserves the same fragment-level ownership and conflict rules as an explicit toggle. Whole-file restoration can erase later user edits, while deleting evidence first makes a safe retry impossible. +- 장점, 단점 및 영향: Successful uninstall retires every recorded client contribution before deleting OpenCodex state. Cleanup is not a cross-client transaction: earlier successful disables remain applied if a later client refuses, and failed writer compensation may leave an intermediate client file. A refusal retains remaining recovery state, not a guarantee that files are unchanged or restoration completed. Operators must inspect reported client files and retained snapshots before retrying. Cleanup takes the existing writer locks and model-roster load before local state is deleted. diff --git a/structure/decisions/ADR-0108-remote-workspace-rpc-deadlines.md b/structure/decisions/ADR-0108-remote-workspace-rpc-deadlines.md new file mode 100644 index 00000000000..7514b6d4a42 --- /dev/null +++ b/structure/decisions/ADR-0108-remote-workspace-rpc-deadlines.md @@ -0,0 +1,12 @@ +# ADR-0108 — decision recorded under "Remote Workspace" + +- Contract owner: [remote-workspace.md](../remote-workspace.md) + +## Decision record + +- 목적과 의도: Ensure a coordinator timeout cannot leave a queued workspace mutation authorized to run later, while allowing the documented 60-second exec ceiling to return normally. +- 기존 구현 및 제약 조건: The coordinator discarded only its pending result after 30 seconds. Executor operations serialize behind one queue, and their abort controllers previously lived only at the endpoint with no request deadline or timeout signal from the coordinator. +- 검토한 주요 대안: Delete late responses only; give each tool an independent queue; use an absolute wall-clock timestamp; send cancellation alone; or combine a bounded relative lifetime with an authenticated cancel frame. +- 선택한 방식: Carry the transport timeout on every encrypted request, start an endpoint abort timer on receipt, check the signal after dequeue through the existing executor boundary, and send a best-effort encrypted cancel frame when the coordinator timer fires. Set the default transport window to 65 seconds and cap negotiated values at 120 seconds. +- 다른 대안 대신 이 방식을 선택한 이유: Relative lifetimes avoid cross-device clock assumptions and cover cancellation frames that are delayed or lost. The cancel frame shortens active work when delivery succeeds, while the request deadline independently prevents queued post-timeout writes. +- 장점, 단점 및 영향: Timed-out queued mutations do not execute, supported commands can use their full 60-second limit, and timeout text no longer claims confirmed cancellation. A non-cooperative running command still depends on its runner honoring AbortSignal, and mixed implementations fail closed rather than silently accepting a request without a lifetime. diff --git a/structure/decisions/ADR-0109-kiro-login-rollback-ownership.md b/structure/decisions/ADR-0109-kiro-login-rollback-ownership.md new file mode 100644 index 00000000000..0934276ef35 --- /dev/null +++ b/structure/decisions/ADR-0109-kiro-login-rollback-ownership.md @@ -0,0 +1,12 @@ +# ADR-0109 — decision recorded under "Forced-login credential rollback" + +- Contract owner: [providers/kiro.md](../providers/kiro.md#forced-login-credential-rollback) + +## Decision record + +- 목적과 의도: Compensate a failed forced Kiro login without deleting credentials or reversing account selection created by another concurrent login. +- 기존 구현 및 제약 조건: Rollback snapshotted only the pre-login account IDs, then deleted every later ID and unconditionally reselected the old active account. Auth-store mutations are already serialized and credentials have stable generation hashes and selection revisions. +- 검토한 주요 대안: Serialize the entire browser login; replace the whole prior provider set; keep the ID-difference rollback; or return an exact write receipt and compensate it with generation and selection checks. +- 선택한 방식: The receipt-bearing credential writer returns the exact written account, generation, post-write selection revision, previous active ID, and previous slot. Rollback performs one serialized mutation, acts only while that generation still matches, preserves a later selection of the same slot, and otherwise removes a created slot or restores only the slot this login replaced. +- 다른 대안 대신 이 방식을 선택한 이유: A global login lock would span interactive authentication and block unrelated work; whole-set or ID-difference restoration cannot distinguish this login from a concurrent successful one. Existing generation and revision metadata provides a narrow ownership proof. +- 장점, 단점 및 영향: Concurrent accounts and refreshes survive a failed publication, while uncontended failures still restore the prior state. A later write to the same slot deliberately wins and can leave the failed login credential present if ownership is no longer provable. diff --git a/structure/decisions/ADR-0110-chat-reasoning-failover-intent.md b/structure/decisions/ADR-0110-chat-reasoning-failover-intent.md new file mode 100644 index 00000000000..7b5e5fff3d8 --- /dev/null +++ b/structure/decisions/ADR-0110-chat-reasoning-failover-intent.md @@ -0,0 +1,12 @@ +# ADR-0110 — decision recorded under "Responses Failover" + +- Contract owner: [transports/responses-failover.md](../transports/responses-failover.md) + +## Decision record + +- 목적과 의도: Preserve a Chat caller's reasoning effort across combo and policy failover while still omitting unsupported controls from each concrete upstream. +- 기존 구현 및 제약 조건: Chat ingress translated one shared Responses body, then stripped effort using the provisional settled route. Combo and policy dispatch already clone and normalize that body per target with the target's own reasoning ladder. +- 검토한 주요 대안: Keep ingress stripping; restore effort from the original Chat body after each failure; attach a second private metadata field; or skip ingress stripping for unresolved routes and use the existing per-target normalizer. +- 선택한 방식: Apply ingress empty-ladder stripping only to a non-combo, non-policy concrete route. Combo and policy retain the translated reasoning object and normalize an isolated child body for every attempt. +- 다른 대안 대신 이 방식을 선택한 이유: The shared body is caller intent, not an attempt wire shape. Reconstructing after mutation is lossy and a second metadata channel can drift, while the existing child normalizer already owns target capability mapping. +- 장점, 단점 및 영향: A capable fallback receives the requested effort even when an earlier target has an empty ladder; unsupported targets still omit it. The unresolved body retains the small reasoning object until a target is chosen. diff --git a/structure/decisions/ADR-0111-legacy-chat-function-history.md b/structure/decisions/ADR-0111-legacy-chat-function-history.md new file mode 100644 index 00000000000..55b5c3b7c90 --- /dev/null +++ b/structure/decisions/ADR-0111-legacy-chat-function-history.md @@ -0,0 +1,12 @@ +# ADR-0111 — decision recorded under "Chat Compatibility" + +- Contract owner: [providers/chat-compat.md](../providers/chat-compat.md) + +## Decision record + +- 목적과 의도: Preserve complete legacy Chat function declarations, assistant calls, and textual results when translating to the Responses protocol. +- 기존 구현 및 제약 조건: Modern `tools` and `tool_calls` were translated, but top-level `functions`, assistant `function_call`, and text `role: function` messages were omitted. Legacy calls carry no call ID, while Responses requires one and downstream adapters require call/result adjacency. +- 검토한 주요 대안: Reject every legacy request; translate declarations only; infer results by transcript position alone; or assign local call IDs and pair pending results by their declared function name. +- 선택한 방식: Translate legacy declarations into function tools, legacy selection into `tool_choice`, assign bounded sequential call IDs to assistant calls, and resolve each textual function result against the pending same-name call. Orphans and malformed shapes fail explicitly; legacy image results retain their existing explicit refusal. +- 다른 대안 대신 이 방식을 선택한 이유: Declaration-only translation still loses executed history, while silent positional pairing can attach a result to the wrong call. Name-bound pending calls preserve the legacy contract without inventing provider identity. +- 장점, 단점 및 영향: Responses providers receive the full executed exchange and no text result disappears. Synthetic IDs are request-local, and ambiguous or orphaned legacy histories now return a clear client error instead of being forwarded incompletely. diff --git a/structure/decisions/ADR-0112-chat-collector-unicode-accounting.md b/structure/decisions/ADR-0112-chat-collector-unicode-accounting.md new file mode 100644 index 00000000000..aa0f2124ad1 --- /dev/null +++ b/structure/decisions/ADR-0112-chat-collector-unicode-accounting.md @@ -0,0 +1,12 @@ +# ADR-0112 — decision recorded under "Stream-buffer accounting" + +- Contract owner: [transports/byte-accounting.md](../transports/byte-accounting.md#stream-buffer-accounting) + +## Decision record + +- 목적과 의도: Keep buffered Chat collector retention equal to the UTF-8 size of its completed strings when a surrogate pair is split across streamed deltas. +- 기존 구현 및 제약 조건: Incremental accounting avoids re-encoding the growing prefix, but it subtracted a fixed two bytes when joining a pair. Bun versions differ in how `Buffer.byteLength` prices each isolated surrogate, so the constant was correct on one runtime and undercounted on another. +- 검토한 주요 대안: Re-encode every accumulated string; standardize on one Bun-specific constant; reject split pairs; or compute the local difference between separate and joined measurements. +- 선택한 방식: Add the runtime-measured joined size and subtract the runtime-measured size of the two isolated code units while retaining the existing prefix byte count. +- 다른 대안 대신 이 방식을 선택한 이유: The differential is constant work, preserves exact full-string semantics, and carries no runtime-version table. Prefix re-encoding would make fragmented streams quadratic. +- 장점, 단점 및 영향: Content, reasoning, and refusal budgets neither undercount nor overcount split Unicode across supported Bun versions. Each boundary performs three tiny byte-length measurements only when it actually joins a surrogate pair. diff --git a/structure/decisions/ADR-0121-remote-workspace-execution-grants.md b/structure/decisions/ADR-0121-remote-workspace-execution-grants.md new file mode 100644 index 00000000000..3bf7043f733 --- /dev/null +++ b/structure/decisions/ADR-0121-remote-workspace-execution-grants.md @@ -0,0 +1,12 @@ +# ADR-0121 — decision recorded under "Remote Workspace" + +- Contract owner: [remote-workspace.md](../remote-workspace.md) + +## Decision record + +- 목적과 의도: Prevent a request whose prepare send remains backpressured through coordinator timeout from acquiring execution authority when those bytes arrive later. +- 기존 구현 및 제약 조건: ADR-0108 starts a relative executor lifetime upon request receipt. Delayed delivery can restart that lifetime after the coordinator stops waiting, before the ordered cancellation arrives. Device wall clocks are not assumed synchronized. +- 검토한 주요 대안: Absolute timestamps require clock assumptions; cancellation alone loses the delayed-receipt race; immediate execution cannot distinguish a still-pending coordinator from an expired one. +- 선택한 방식: RPC v2 separates authenticated prepare from grant. Prepare validates and retains bounded request state but cannot invoke. After prepare send completion, a grant is admitted only while the same pending request remains live, with the check inside the serialized send queue. Cancellation and expiry discard ungranted state; the existing abort signal owns granted work. RPC v1 is rejected without downgrade; encrypted framing is unchanged. +- 다른 대안 대신 이 방식을 선택한 이유: The grant check closes the prepare-backpressure race without a shared clock, preserves directional encryption order, and prevents old immediate-execution endpoints from silently bypassing the new contract. +- 장점, 단점 및 영향: Timeout delivery no longer waits for a blocked send. Prepare-only requests never enter the executor. This adds one authenticated message and requires both peers to upgrade. A grant already sent can itself be delayed, and a running operation may already have committed; timeout remains an unknown outcome with cancellation requested, not proof of rollback or universal post-timeout non-execution. diff --git a/structure/manifest.json b/structure/manifest.json index a4349938e8e..734e7d71aa8 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -157,6 +157,7 @@ "scope": "Request-copy and stream-buffer byte accounting shared by parsing, SSE rewriting, the adapters, and the translator budget.", "documents": [ "src/adapters/", + "src/chat/", "src/lib/", "src/server/" ] @@ -346,6 +347,7 @@ "scope": "Cross-vendor Chat Completions behavior: reasoning, tool results, structured output, parallel tools.", "documents": [ "src/adapters/", + "src/chat/", "src/responses/" ] }, @@ -414,6 +416,7 @@ "title": "Client Integrations", "scope": "Third-party client config ownership, snapshots, refresh, disable, and restore.", "documents": [ + "src/cli/", "src/clients/", "src/integrations/", "src/lib/" diff --git a/structure/ops/service-and-sidecars.md b/structure/ops/service-and-sidecars.md index d75f8e3bca6..2c5b32d69b0 100644 --- a/structure/ops/service-and-sidecars.md +++ b/structure/ops/service-and-sidecars.md @@ -244,10 +244,18 @@ identity and proven-dead liveness; unknown or transferred ownership never starts Direct recovery retains the lease until readiness or its bounded deadline. The normal successful manual-runtime update still prints the existing restart hint. +The npm launcher in `bin/ocx.mjs` makes one exception after a failed update: a service recovery +releases the lease before the service refresh, as a successful update does. The service manager +starts the proxy outside the updater's process tree, so that proxy has to take the lease itself; +held through the repair's health wait, the lease kept it from starting, and recovery fell through +to a second, directly started proxy (#5760). The recovery decision is made again after the release. + The npm transaction creates each staging directory exclusively and may clean that fresh path -while the creating process still owns it. A later update only reports staging leftovers. It does -not recursively delete them from a marker: the marker is not an authorization secret, and a -neighbouring writer could replace a previously checked pathname with a link before traversal. +while the creating process still owns it. On POSIX it also creates the stage's `lib` directory, +because npm's strict script policy plans the global tree before it creates the prefix layout +(#5760). A later update only reports staging leftovers. It does not recursively delete them +from a marker: the marker is not an authorization secret, and a neighbouring writer could +replace a previously checked pathname with a link before traversal. The probe ceilings are module-load constants in `src/server/proxy-liveness.ts`: 750 ms for the shared default and 1500 ms (three attempts) for `SERVICE_STOP_LIVENESS` and diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 6597e31bede..2058602ca09 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -80,7 +80,7 @@ rewrite rules and the routed-id settlement. | `src/adapters/declaration-carrier.ts`, `src/adapters/input-media-guard.ts` | Default-deny allowlists for constraints the normalized request carries but a wire may not be able to express: `tools[*].allowed_callers`, which fences a tool off from callers, and inline document bytes. Both are refused with a 400 at the single guard every registered adapter passes through, rather than left to each adapter, because an adapter that never learned about the carrier rebuilds without it and answers normally. `allowed_callers` reaches the `anthropic` wire; document bytes reach `anthropic`, `openai-chat` and `google`; the `openai-responses` wire is exempt from the whole guard because it forwards the original body. Adding an `AdapterWire` member makes the omission visible in these lists instead of at a customer's upstream. The unrestricted `["direct"]` caller default is not a restriction. | | `src/adapters/azure.ts` | Azure OpenAI bridge. | | `src/adapters/cursor.ts`, `src/adapters/cursor/` | Cursor protobuf transport: discovery, request builder, event decoding, MCP, thread continuity, native-exec policy. | -| `src/adapters/devin.ts`, `src/adapters/devin/cloud-direct/` | Devin runTurn transport over Cognition Connect-RPC. `GetChatMessage` uses the Responses provider executor and shared physical-send budget; catalog and JWT support RPCs remain outside inference-send accounting. Provider-stated 429 reset delays are surfaced to the client rather than slept inside an admitted turn, so they cannot retain shared active-turn capacity. A recorded tenant host is used only for the stored account whose credential owns the transmitted key, searched in the configured provider id and then its deprecated alias; a configured, forwarded, or unmatched key uses the configured base URL or the US default. | +| `src/adapters/devin.ts`, `src/adapters/devin/cloud-direct/` | Devin runTurn transport over Cognition Connect-RPC. `GetChatMessage` uses the Responses provider executor and shared physical-send budget; catalog, JWT, and `src/web-search/devin-executor.ts` native search support RPCs remain outside inference-send accounting. Provider-stated 429 reset delays are surfaced to the client rather than slept inside an admitted turn, so they cannot retain shared active-turn capacity. A recorded tenant host is used only for the stored account whose credential owns the transmitted key, searched in the configured provider id and then its deprecated alias; a configured, forwarded, or unmatched key uses the configured base URL or the US default. Native search previews the current route by effective adapter without mutating combo selection state, pins one admitted active-account snapshot for the request, and calls `GetWebSearchResults`, so it starts no CLI or second model. | | `src/adapters/kiro.ts` and `src/adapters/kiro/` | Kiro event/tool/thinking/truncation/retry handling. The original path is a facade over leaves for wire identity, reasoning, conversation state, token estimation, payload assembly, streaming, and the adapter. | | `src/adapters/mimo-free.ts` | Mimo Free transport (client identity + JWT). Concurrent requests share one JWT bootstrap bound only to its timeout; each request stops waiting on its own abort without cancelling the others. | | `src/adapters/command-code.ts`, `src/adapters/command-code-tool-text.ts`, `src/adapters/command-code-restored-schema.ts` | Command Code OAuth NDJSON translation. For every `xiaomi/mimo-` model, text, native calls, reasoning, and terminal decisions share one byte-bounded queue with linear queue visits. Markup is deduplicated against matching native calls; text-only restoration requires one contiguous text run, a clean finish, a declared tool, and arguments validated against supported schema constraints. A parameter-free (freeform) block may omit `` but must end with ``; parameter blocks keep the canonical close. Markup appended after prose in the same delta is split off at the marker and held like a block that opens with ``; a marker split across deltas after prose is still released as text. Native, reasoning, and other intervening events interrupt a still-probing block but leave a held block held in arrival order, and the queued byte bound still flushes an unresolved envelope as text. An envelope the strict parser rejects but that opens with ``, closes with ``, and names a declared function is dropped when a native call for that same function arrives and on a clean finish; markup that parses but fits no supported schema is still released as text. Regex patterns, other unsupported constraints, and abnormal finishes fail closed. `tests/providers/command-code-tool-text-prose-split.test.ts` covers the split, the interleaved-event hold, and both drop paths. | diff --git a/structure/providers/chat-compat.md b/structure/providers/chat-compat.md index a721794ee23..02d928fcd4a 100644 --- a/structure/providers/chat-compat.md +++ b/structure/providers/chat-compat.md @@ -498,6 +498,14 @@ reasoning. > Decision record: [ADR-0068](../decisions/ADR-0068-reasoning-display-parity-hidethinkingsummary.md) +`src/chat/inbound.ts` translates legacy Chat `functions`, assistant `function_call`, and textual +`role: "function"` results as one Responses function exchange. Missing or null assistant +`function_call` fields mean no call and preserve ordinary assistant text. The translator assigns +bounded sequential call IDs and pairs results by function name; malformed or orphan results fail +explicitly, and image-bearing legacy results remain unsupported rather than losing media. + +> Decision record: [ADR-0111](../decisions/ADR-0111-legacy-chat-function-history.md) + ## Chat streamed tool-call identity `src/adapters/openai-chat.ts` retains a call's first observed non-negative safe integer diff --git a/structure/providers/kiro.md b/structure/providers/kiro.md index b87b0274abd..75551b579fd 100644 --- a/structure/providers/kiro.md +++ b/structure/providers/kiro.md @@ -20,6 +20,16 @@ shared POSIX bin directories (`~/.local/bin`, `/usr/local/bin`, `/opt/homebrew/b unrelated `kiro` such as the Kiro IDE launcher can live. Coverage: `tests/providers/kiro/kiro-windows-cli-executable-path.test.ts`. +## Forced-login credential rollback + +A forced login uses a receipt-bearing auth-store write naming its exact account, credential +generation, selection revision, and prior slot. If later provider publication fails, rollback is +one serialized compare-and-swap mutation: it removes or restores only that still-owned generation. +A concurrent account addition, selection, or credential refresh wins and is never inferred from a +before/after account-ID set. + +> Decision record: [ADR-0109](../decisions/ADR-0109-kiro-login-rollback-ownership.md) + ## Kiro client parallel-tool hint Kiro's wire remains serialized even when an OpenAI Responses client sends diff --git a/structure/remote-workspace.md b/structure/remote-workspace.md index c9c639219a9..467b603cfa1 100644 --- a/structure/remote-workspace.md +++ b/structure/remote-workspace.md @@ -6,6 +6,21 @@ `src/remote-control/workspace-agent-connection.ts` intersects presence with enrollment authority and negotiates explicit session grants. `src/remote-control/workspace-rpc.ts` snapshots session/device/root/capabilities and rejects mismatches before invoking the executor. The paired Hub is trusted to select an approved root over authenticated WSS; workspace control traffic is not an untrusted opaque relay protocol. +Encrypted RPC v2 prepares a request with a bounded executor lifetime without invoking it. Only a +separate authenticated grant admits execution. The coordinator sends that grant after prepare +delivery settles, checking the original pending request again when the serialized grant send starts. +A prepare whose send is still backpressured at coordinator timeout therefore cannot execute later. +The endpoint starts its relative deadline on prepare receipt, never resets it on grant, and removes +ungranted requests on cancellation or expiry. Granted work receives the same abort signal through +the executor queue. The default 65-second RPC window exceeds the supported 60-second command ceiling. +Timeout requests cancellation but does not confirm it: an already-sent grant can still be delayed +in transit or its operation can already be running. The wire framing and encryption are unchanged; +RPC v1 peers fail closed and must upgrade together rather than fall back to immediate execution. + +> Decision record: [ADR-0108](decisions/ADR-0108-remote-workspace-rpc-deadlines.md) + +> Decision record: [ADR-0121](decisions/ADR-0121-remote-workspace-execution-grants.md) + `src/remote-control/workspace-executor.ts` checks approved root identity, relative paths, file size and write preconditions. File reads and write preconditions open descriptors nonblocking before verifying regular-file identity, so special files cannot wait for a peer during open. Its optional command runner lives in `src/remote-control/workspace-command-runner.ts`. Linux uses bubblewrap outside writable workspace roots and checks executable/parent permissions before invocation. The official Windows and macOS native helpers refuse commands; file tools remain independent of command availability. `src/remote-control/workspace-hub.ts`, `src/remote-control/workspace-device.ts` and `src/remote-control/workspace-sessions.ts` own separate persisted state. `src/remote-control/workspace-secret-store.ts` requires private permissions and rejects access failures rather than treating them as first-run absence. Publication reuses `src/config/atomic-write.ts`; workspace file publication uses the remote-workspace publisher in `src/lib/windows-atomic-replace.ts`. diff --git a/structure/transports/byte-accounting.md b/structure/transports/byte-accounting.md index 6021f7cff98..f514bcdcb5e 100644 --- a/structure/transports/byte-accounting.md +++ b/structure/transports/byte-accounting.md @@ -86,6 +86,13 @@ Serialized request and buffered-response observations use byte counts without me The same rule applies to Anthropic, Google, and Chat response accounting; serialization itself is preserved where the existing metric is the serialized JSON size. +The buffered Chat collector in `src/chat/outbound.ts` computes a split surrogate pair's incremental +UTF-8 cost from the runtime's measured separate and joined sizes. It does not assume how a Bun +version prices a lone surrogate, so retained content, reasoning, and refusal fields enforce and +release the same exact budget on every supported runtime. + +> Decision record: [ADR-0112](../decisions/ADR-0112-chat-collector-unicode-accounting.md) + `src/lib/translator-budget.ts` admits an event batch atomically from per-event serialized byte sizes plus exact separators, without joining a second full JSON array. `src/lib/admission.ts` counts and truncates diagnostic text at UTF-8 code-point boundaries without allocating arrays per character; diff --git a/structure/transports/inventory.md b/structure/transports/inventory.md index cf43a7a9491..6e08a4b846e 100644 --- a/structure/transports/inventory.md +++ b/structure/transports/inventory.md @@ -37,7 +37,7 @@ surface is listed here so a maintainer can find the owner without grepping: | Cursor (beyond the sections above) | `src/adapters/cursor/live-transport.ts`, `src/adapters/cursor/http1-bidi.ts`, `src/adapters/cursor/live-models.ts`, `src/adapters/cursor/transport-retry.ts`, `src/adapters/cursor/mcp-manager.ts`, `src/adapters/cursor/thread-continuity.ts`, `src/adapters/cursor/checkpoint-store.ts` | Thread continuity is the point: a retry must not start a new Cursor thread, and a validated checkpoint must not rebuild the full root history. HTTP/2 remains the default; an explicit `http1.1`/`h1` pin maps the bidi run onto Cursor's `RunSSE` receive stream plus sequenced `BidiAppend` sends, and applies to live discovery too. | | Claude Messages | `src/server/claude-messages.ts` | Routed translation, a native Anthropic passthrough branch, and `count_tokens`. | | Chat Completions inbound | `src/server/chat-completions.ts`, `src/server/chat-native.ts`, `src/chat/`, `src/adapters/openai-chat.ts` | Inbound translation onto the same routing pipeline. The content mapper preserves image URLs and supported detail, including screenshot-bearing tool results; target adapters own image placement on their wire. Image-free tool results stay strings. The native handler owns pin/cap normalization; both adapter builders share explicit gateway-object and tool-bearing effort-omission policy, while the native builder preserves unknown or undeclared raw behavior and removes effort for explicit empty declarations or no-reasoning models. On the response side, the upstream `service_tier` echo relays on every delivery shape (`src/chat/outbound.ts` projections, `src/server/chat-native-sse.ts` chunks); an upstream without the field gets no injected key. | -| Hosted search relay | `src/server/search.ts` | Verbatim ChatGPT relay, or an explicitly configured web-search sidecar backend when no forward provider exists; distinct from the web-search sidecar loop below. | +| Hosted search relay | `src/server/search.ts`, `src/web-search/devin-executor.ts` | A resolved Devin route uses Cognition's bounded non-inference `GetWebSearchResults` RPC with the active account's credential and allowlisted tenant; other routes use the verbatim ChatGPT relay, or an explicitly configured web-search sidecar backend when no forward provider exists. Distinct from the web-search sidecar loop below. | | Image/video generation loop | `src/images/loop.ts`, `src/images/plan.ts`, `src/images/fulfill.ts`, `src/images/xai-client.ts`, `src/images/xai-video-client.ts`, `src/images/artifacts.ts` | A provider-returned image URL is downloaded into a local artifact once, then served locally; warnings stay URL-free because provider CDN URLs may embed credentials. Artifact downloads go through the pinned-IP transport with a 10 s connect deadline (`DOWNLOAD_CONNECT_TIMEOUT_MS`) that bounds TCP/TLS setup on its own, in addition to the 60 s idle timer, and `pinnedHttpsGet` accepts a per-call `connectTimeoutMs`. | | GitHub Copilot | `src/providers/xai-transport.ts` (`resolveProviderTransport`), `src/providers/github-copilot-transport.ts` | `resolveProviderTransport` selects the Copilot transport when the routed provider name is `github-copilot`; the Copilot module then resolves its headers and base URL, and the registry seeds the provider row and model fallback. | | API-key pools | `src/providers/api-key-selection.ts`, `src/providers/key-failover.ts` | A configured `apiKeyPoolStrategy` plus a cooling committed key rotates before the first send (`selectProactiveApiKeyTransport`); a 429 still rotates after the send and records a cooldown. `provider.apiKey` keeps mirroring the active entry so routing stays single-key. The pick is inert without a strategy or while the committed key is healthy. | diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index f42b4e6f4c8..b31f57d6f45 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -206,6 +206,15 @@ Native Chat applies qualifying effort ceilings independently of model pins; pin Pool quota producers and account commands follow the [bounded raw-observation contract](../providers/openai-accounts.md#bounded-pool-quota-observations), separate from the latest display snapshot and capacity estimates; account quota surfaces use [safe probe diagnostics](inventory.md#account-quota-failure-diagnostics) separately from quota validity, credential health and routing authority. Raw-byte readers on this path supply their own byte and deadline budgets under the [bounded ingestion contract](inventory.md#bounded-response-ingestion-and-orcarouter-login). +Translated Chat requests preserve caller reasoning intent until a combo or policy selects a +concrete target. Empty-ladder stripping and effort mapping apply to each attempt copy, never the +shared ingress body, so a later capable fallback still receives the caller's requested effort. +`src/server/responses/core-normalize.ts` strips an empty ladder from both parsed adapter options +and raw reasoning on each translated Chat attempt, preserving summary controls. Policy fallback +captures the original body before this normalization, including for its first candidate. + +> Decision record: [ADR-0110](../decisions/ADR-0110-chat-reasoning-failover-intent.md) + Live sideband admission and its bounded upstream handshake follow the [runtime contract](../runtime.md#live-sideband-handshake); the ordinary Responses WebSocket exchange remains separate. Translated Chat request construction uses the [inline-image budget](streaming-health.md#translated-chat-inline-image-budget); the shared normalizer counts retained bytes even when a wire-specific drop callback keeps the image attached, rejects inputs above the safe decoded-pixel ceiling, caps native decode work process-wide, and stops queued work when the request is cancelled. diff --git a/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts b/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts index 7328d081f42..b4686b8ad5c 100644 --- a/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts +++ b/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts @@ -9,6 +9,7 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AdapterRequest, IncomingMeta, ProviderAdapter } from "../../../src/adapters/base"; +import type { AttemptRecoveryKind } from "../../../src/usage/log"; import { clearAnthropicAccountPoolState } from "../../../src/oauth/anthropic-routing"; import { clearGenericFailoverHealth } from "../../../src/oauth/generic-account-failover"; import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; @@ -71,7 +72,9 @@ beforeAll(async () => { adapter: ProviderAdapter; incomingMeta: IncomingMeta; fetchForRequest: (request: AdapterRequest, parsed: OcxParsedRequest) => typeof fetch; - on429?: (retryAfter: string | null) => Promise; + on429?: (retryAfter: string | null) => Promise< + { adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null + >; }) => { // This is a dispatch seam test. The real loop is covered in anthropic-quota-dispatch. const first = await args.adapter.buildRequest(args.parsed, args.incomingMeta); @@ -83,7 +86,11 @@ beforeAll(async () => { await refused.body?.cancel(); const rotated = await args.on429?.(retryAfter); if (!rotated) throw new Error("Anthropic sidecar did not rotate after 429"); - const second = await rotated.buildRequest(args.parsed, args.incomingMeta); + // Unwrapped exactly as the real loop does. This seam drives the PRODUCTION rotator + // (`rotateSidecarProviderOn429`), so it is the one place the Anthropic arm's kind is + // proven end to end rather than against a hand-written stub. + expect(rotated.recoveryKind).toBe("anthropic-oauth-429"); + const second = await rotated.adapter.buildRequest(args.parsed, args.incomingMeta); return args.fetchForRequest(second, args.parsed)(second.url, { method: second.method, headers: second.headers, body: second.body, }); diff --git a/tests/cli/uninstall.test.ts b/tests/cli/uninstall.test.ts index 1b0e6f399b5..74d99e1429a 100644 --- a/tests/cli/uninstall.test.ts +++ b/tests/cli/uninstall.test.ts @@ -7,11 +7,17 @@ import { pathToFileURL } from "node:url"; import { repoRoot } from "../helpers/repo-root"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { removeOwnedConfigAfterDesktopCleanup, type UninstallClientStateDeps } from "../../src/cli/uninstall-client-state"; +import { cleanupOwnedIntegrationsBeforeUninstall } from "../../src/cli/uninstall-integrations"; import { assertClientLifecycleHeld, withClientLifecycle, withClientLifecycleSync, type ClientLifecycleHeld } from "../../src/client/lifecycle-lock"; import type { UninstallObservation } from "../../src/cli/uninstall-plan"; import type { DesktopDisconnectReceipt } from "../../src/claude/desktop-remote-store"; +import type { ExportModel } from "../../src/clients/config-export"; +import { INTEGRATION_CLIENTS } from "../../src/integrations/registry"; +import { createIntegrationStateStore } from "../../src/integrations/store"; +import { applyIntegration, disableIntegrationCoordinated } from "../../src/integrations/writer"; +import type { OcxConfig } from "../../src/types"; const root = pathToFileURL(repoRoot() + "/"); @@ -321,16 +327,17 @@ function uninstallFixture() { beforeFinalLock?: () => void; beforeDisconnectLock?: () => void; duringCleanup?: () => Promise; + duringIntegrationCleanup?: () => Promise; duringRemove?: () => void; finishCleanup: boolean; lease?: ClientLifecycleHeld; aclReapPending: boolean; - calls: { read: number; cleanup: number; remove: number; finalLock: number }; + calls: { read: number; cleanup: number; integrations: number; remove: number; finalLock: number }; cleanupOptions: Array[0]>; } = { connection: { kind: "disconnected" }, desktop: { kind: "absent" }, receipt: { kind: "absent" }, finishCleanup: true, aclReapPending: false, - calls: { read: 0, cleanup: 0, remove: 0, finalLock: 0 }, cleanupOptions: [], + calls: { read: 0, cleanup: 0, integrations: 0, remove: 0, finalLock: 0 }, cleanupOptions: [], }; const deps: UninstallClientStateDeps = { readConnection: () => { fixture.calls.read++; return fixture.connection; }, @@ -366,6 +373,11 @@ function uninstallFixture() { finally { fixture.lease = undefined; } }, { lockPath }); }, + cleanupIntegrations: async () => { + fixture.calls.integrations++; + await fixture.duringIntegrationCleanup?.(); + return { attempted: 0, changed: 0 }; + }, remove: () => { // Real destructive work is confined to this fixture, never getConfigDir(). assertClientLifecycleHeld(fixture.lease!); @@ -393,7 +405,7 @@ describe("uninstall client cleanup before owner-state deletion", () => { const before = f.bytes(); await expect(removeOwnedConfigAfterDesktopCleanup({ ...safeTeardown, proxyProvenDown: false }, f.deps)) .rejects.toThrow("teardown is not proven"); - expect(f.fixture.calls).toEqual({ read: 0, cleanup: 0, remove: 0, finalLock: 0 }); + expect(f.fixture.calls).toEqual({ read: 0, cleanup: 0, integrations: 0, remove: 0, finalLock: 0 }); expect(f.bytes()).toEqual(before); expect(existsSync(f.lockPath)).toBe(false); }); @@ -463,6 +475,18 @@ describe("uninstall client cleanup before owner-state deletion", () => { }); }); + test("an integration cleanup failure preserves OpenCodex recovery state and skips removal", async () => { + await withUninstallFixture(async f => { + const before = f.bytes(); + f.fixture.duringIntegrationCleanup = async () => { throw new Error("fixture integration conflict"); }; + await expect(removeOwnedConfigAfterDesktopCleanup(safeTeardown, f.deps)) + .rejects.toThrow("fixture integration conflict"); + expect(f.fixture.calls.integrations).toBe(1); + expect(f.fixture.calls.remove).toBe(0); + expect(f.bytes()).toEqual(before); + }); + }); + test("a replacement connection before disconnect claims L survives uninstall", async () => { await withUninstallFixture(async f => { f.fixture.connection = connectedFixture; @@ -551,6 +575,7 @@ describe("uninstall client cleanup before owner-state deletion", () => { }; expect(await removeOwnedConfigAfterDesktopCleanup(safeTeardown, f.deps)).toEqual({ status: "removed", residualPaths: [] }); expect(f.fixture.calls.cleanup).toBe(mode === "connected" ? 1 : 0); + expect(f.fixture.calls.integrations).toBe(1); expect(f.fixture.calls.remove).toBe(1); expect(existsSync(f.configDir)).toBe(false); expect(existsSync(f.lockPath)).toBe(true); // L is outside the directory being removed. @@ -559,3 +584,172 @@ describe("uninstall client cleanup before owner-state deletion", () => { }); }); }); + +describe("uninstall restores recorded third-party integrations before deleting recovery state", () => { + const models: ExportModel[] = [ + { namespaced: "fixture/model", provider: "fixture", id: "model", contextWindow: 128_000 }, + ]; + const config = { + port: 10100, + hostname: "127.0.0.1", + defaultProvider: "fixture", + providers: { fixture: { adapter: "openai-chat", baseUrl: "http://127.0.0.1/v1" } }, + } as unknown as OcxConfig; + + async function fixture() { + const root = mkdtempSync(join(tmpdir(), "ocx-uninstall-integrations-")); + const home = join(root, "home"); + const configDir = join(root, "opencodex"); + const lockPath = join(root, "runtime", "lifecycle.sqlite"); + const env = {} as NodeJS.ProcessEnv; + mkdirSync(INTEGRATION_CLIENTS.pi.detectDir(env, home), { recursive: true }); + mkdirSync(configDir, { recursive: true }); + const clientConfig = INTEGRATION_CLIENTS.pi.configPath(env, home); + mkdirSync(dirname(clientConfig), { recursive: true }); + writeFileSync(clientConfig, '{"userSetting":"keep"}\n'); + const store = createIntegrationStateStore(join(configDir, "integrations")); + const input = { clientId: "pi" as const, models, config, port: config.port, env, home, store }; + expect(applyIntegration(input).ok).toBe(true); + const deps: UninstallClientStateDeps = { + readConnection: () => ({ kind: "disconnected" }), + inspectDesktop: () => ({ kind: "absent" }), + readReceipt: () => ({ kind: "absent" }), + disconnect: async () => undefined, + withLifecycle: work => withClientLifecycle(work, { lockPath }), + cleanupIntegrations: () => cleanupOwnedIntegrationsBeforeUninstall({ + createStore: () => store, + loadConfig: () => config, + loadModels: async () => models, + disable: value => disableIntegrationCoordinated(value), + env, + home, + }), + remove: () => { + rmSync(configDir, { recursive: true }); + return { status: "removed", residualPaths: [] }; + }, + aclReapPending: () => false, + }; + return { root, home, env, configDir, clientConfig, store, deps }; + } + + function addAsideProfiles(f: Awaited>) { + const asideRoot = join(f.home, ".aside"); + const profiles = [0, 1].map(id => { + const detectDir = join(asideRoot, "u", String(id)); + const configPath = join(detectDir, "models.json"); + mkdirSync(detectDir, { recursive: true }); + writeFileSync(configPath, '{"userSetting":"keep"}\n'); + const store = id === 0 ? f.store + : createIntegrationStateStore(join(f.store.root, "aside-profiles", String(id))); + return { id, detectDir, configPath, store }; + }); + // The legacy owner is not the current profile; uninstall must use its recorded path. + writeFileSync(join(asideRoot, "accounts.json"), JSON.stringify({ + currentAccountId: 1, accounts: profiles.map(({ id }) => ({ id })), + })); + for (const profile of profiles) { + expect(applyIntegration({ clientId: "aside", models, config, port: config.port, + env: f.env, home: f.home, store: profile.store, + resolvedPaths: { configPath: profile.configPath, detectDir: profile.detectDir }, + }).ok).toBe(true); + } + return profiles; + } + + test("restores the external file before removing the records that authorize restoration", async () => { + const f = await fixture(); + try { + expect(await removeOwnedConfigAfterDesktopCleanup(safeTeardown, f.deps)) + .toEqual({ status: "removed", residualPaths: [] }); + expect(existsSync(f.configDir)).toBe(false); + const restored = JSON.parse(readFileSync(f.clientConfig, "utf8")) as Record; + expect(restored.userSetting).toBe("keep"); + expect((restored.providers as Record | undefined)?.opencodex).toBeUndefined(); + } finally { + rmSync(f.root, { recursive: true, force: true }); + } + }); + + test("a conflicting external edit retains both the file and recovery records", async () => { + const f = await fixture(); + try { + const edited = JSON.parse(readFileSync(f.clientConfig, "utf8")) as { + providers: Record>; + }; + edited.providers.opencodex!.baseUrl = "http://user-edited.invalid/v1"; + writeFileSync(f.clientConfig, `${JSON.stringify(edited, null, 2)}\n`); + await expect(removeOwnedConfigAfterDesktopCleanup(safeTeardown, f.deps)) + .rejects.toThrow("integration cleanup refused for pi"); + expect(existsSync(f.configDir)).toBe(true); + expect(f.store.readRecordsStrict().pi).toBeDefined(); + expect(readFileSync(f.clientConfig, "utf8")).toContain("user-edited.invalid"); + } finally { + rmSync(f.root, { recursive: true, force: true }); + } + }); + + test("restores legacy and child Aside profiles before deleting all recovery stores", async () => { + const f = await fixture(); + try { + const profiles = addAsideProfiles(f); + expect(await removeOwnedConfigAfterDesktopCleanup(safeTeardown, f.deps)) + .toEqual({ status: "removed", residualPaths: [] }); + expect(existsSync(f.configDir)).toBe(false); + for (const profile of profiles) { + expect(JSON.parse(readFileSync(profile.configPath, "utf8"))) + .toEqual({ userSetting: "keep" }); + } + } finally { rmSync(f.root, { recursive: true, force: true }); } + }); + + test("an unreadable Aside child record prevents every disable and config removal", async () => { + const f = await fixture(); + try { + const profiles = addAsideProfiles(f); + const originals = profiles.map(profile => readFileSync(profile.configPath, "utf8")); + const childRecords = join(profiles[1]!.store.root, "records.json"); + writeFileSync(childRecords, "invalid JSON"); + await expect(removeOwnedConfigAfterDesktopCleanup(safeTeardown, f.deps)) + .rejects.toThrow("integration ownership is invalid for recovery"); + expect(existsSync(f.configDir)).toBe(true); + expect(f.store.readRecordsStrict().pi).toBeDefined(); + expect(profiles.map(profile => readFileSync(profile.configPath, "utf8"))).toEqual(originals); + expect(readFileSync(childRecords, "utf8")).toBe("invalid JSON"); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }); + + test("a conflicted Aside child retains recovery state without undoing earlier disables", async () => { + const f = await fixture(); + try { + const profiles = addAsideProfiles(f); + const child = profiles[1]!; + const edited = JSON.parse(readFileSync(child.configPath, "utf8")); + edited.providers.opencodex.baseUrl = "http://user-edited.invalid/v1"; + writeFileSync(child.configPath, `${JSON.stringify(edited)}\n`); + await expect(removeOwnedConfigAfterDesktopCleanup(safeTeardown, f.deps)) + .rejects.toThrow("integration cleanup refused for aside"); + expect(existsSync(f.configDir)).toBe(true); + expect(child.store.readRecordsStrict().aside).toBeDefined(); + expect(readFileSync(child.configPath, "utf8")).toContain("user-edited.invalid"); + expect(f.store.readRecordsStrict().pi).toBeUndefined(); + expect(f.store.readRecordsStrict().aside).toBeUndefined(); + expect(JSON.parse(readFileSync(profiles[0]!.configPath, "utf8"))).toEqual({ userSetting: "keep" }); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }); + + test("an unregistered owned Aside profile refuses cleanup rather than dropping its proof", async () => { + const f = await fixture(); + try { + const profiles = addAsideProfiles(f); + writeFileSync(join(f.home, ".aside", "accounts.json"), JSON.stringify({ + currentAccountId: 0, accounts: [{ id: 0 }], + })); + await expect(removeOwnedConfigAfterDesktopCleanup(safeTeardown, f.deps)) + .rejects.toThrow("Aside profile ownership is missing or mismatched"); + expect(existsSync(f.configDir)).toBe(true); + expect(f.store.readRecordsStrict().pi).toBeDefined(); + expect(profiles[1]!.store.readRecordsStrict().aside).toBeDefined(); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }); +}); diff --git a/tests/clients/remote-workspace-session-binding.test.ts b/tests/clients/remote-workspace-session-binding.test.ts index ac1ab7c7c38..04ba60e0d4e 100644 --- a/tests/clients/remote-workspace-session-binding.test.ts +++ b/tests/clients/remote-workspace-session-binding.test.ts @@ -37,13 +37,21 @@ function fixture() { }; return { invocations, - async send(overrides: Partial = {}) { + async send(overrides: Partial = {}, grant = true, version = 2) { const message = new TextEncoder().encode(JSON.stringify({ - version: 1, kind: "request", request: { ...request, ...overrides }, + version, kind: version === 1 ? "request" : "prepare", timeoutMs: 5_000, request: { ...request, ...overrides }, })); for (const frame of frameRemoteWorkspaceRpcMessage(message)) { await endpoint.receiveCiphertext(client.encrypt(frame)); } + if (grant) { + const grantMessage = new TextEncoder().encode(JSON.stringify({ + version, kind: "grant", requestId: overrides.requestId ?? request.requestId, + })); + for (const frame of frameRemoteWorkspaceRpcMessage(grantMessage)) { + await endpoint.receiveCiphertext(client.encrypt(frame)); + } + } }, close() { endpoint.close(); client.destroy(); }, }; @@ -65,6 +73,22 @@ test("encrypted requests cannot leave their session grant before executor invoca } }); +test("an encrypted prepare cannot invoke without an execution grant", async () => { + const state = fixture(); + try { + await state.send({}, false); + expect(state.invocations).toEqual([]); + } finally { state.close(); } +}); + +test("legacy immediate-execution RPC requests fail closed", async () => { + const state = fixture(); + try { + await expect(state.send({}, false, 1)).rejects.toThrow("unsupported remote workspace RPC version"); + expect(state.invocations).toEqual([]); + } finally { state.close(); } +}); + test("a matching encrypted read reaches the selected executor once", async () => { const state = fixture(); try { diff --git a/tests/clients/remote-workspace.test.ts b/tests/clients/remote-workspace.test.ts index a639c848f83..a03d536ec02 100644 --- a/tests/clients/remote-workspace.test.ts +++ b/tests/clients/remote-workspace.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { createHash, randomUUID } from "node:crypto"; import { execFileSync, spawnSync } from "node:child_process"; import { + existsSync, mkdirSync, linkSync, mkdtempSync, @@ -501,4 +502,160 @@ describe("remote workspace coordinator and executor", () => { client.close(); endpoint.close(); }); + + test("a prepare delayed by send backpressure never grants a timed-out mutation", async () => { + const state = fixture(); + const account = generateRemoteControlIdentityKeyPair(); + const device = generateRemoteControlIdentityKeyPair(); + const deviceId = randomUUID(); + const sessionId = randomUUID(); + const handshake = RemoteControlClientHandshake.create({ + sessionId, deviceId, commandProfile: "codex", capabilities: ["workspace.write"], + accountPrivateKey: account.privateKey, + }); + const accepted = acceptRemoteControlClientHello(handshake.hello, { + expectedSessionId: sessionId, expectedDeviceId: deviceId, + accountPublicKey: account.publicKey, devicePrivateKey: device.privateKey, + allowedCapabilities: ["workspace.write"], + }); + let releaseSend!: () => void; + const backpressure = new Promise(resolve => { releaseSend = resolve; }); + let markDrained!: () => void; + const drained = new Promise(resolve => { markDrained = resolve; }); + let endpoint!: EncryptedRemoteWorkspaceExecutorEndpoint; + let sent = 0; + const client = new EncryptedRemoteWorkspaceTransport({ + executorDeviceId: deviceId, cipher: handshake.complete(accepted.hello, device.publicKey), + timeoutMs: 50, + async sendCiphertext(value) { + sent++; + if (sent === 1) await backpressure; + await endpoint.receiveCiphertext(value); + if (sent === 2) markDrained(); + }, + }); + let invocations = 0; + const executor = new RemoteWorkspaceExecutor({ + deviceId, roots: [{ id: "project-root", path: state.executorRoot }], + }); + endpoint = new EncryptedRemoteWorkspaceExecutorEndpoint({ + executorDeviceId: deviceId, sessionId, rootId: "project-root", + capabilities: ["workspace.write"], cipher: accepted.cipher, + executor: { invoke(request, signal) { invocations++; return executor.invoke(request, signal); } }, + sendCiphertext: value => client.receiveCiphertext(value), + }); + const target = join(state.executorRoot, "project", "delayed-prepare.txt"); + try { + await expect(client.invoke({ + requestId: randomUUID(), sessionId, executorDeviceId: deviceId, rootId: "project-root", + tool: "write_file", + arguments: { path: "project/delayed-prepare.txt", content: "must-not-run", expectedSha256: null }, + })).rejects.toThrow("cancellation was requested"); + expect(sent).toBe(1); + releaseSend(); + await drained; + // Let the queued grant admission run after the cancellation send has drained. + await new Promise(resolve => setTimeout(resolve, 0)); + expect(sent).toBe(2); + expect(invocations).toBe(0); + expect(existsSync(target)).toBe(false); + } finally { + releaseSend(); + client.close(); + endpoint.close(); + } + }); + + test("a timed-out mutation queued behind a command is cancelled before dequeue", async () => { + const state = fixture(); + const account = generateRemoteControlIdentityKeyPair(); + const device = generateRemoteControlIdentityKeyPair(); + const cryptoDeviceId = randomUUID(); + const cryptoSessionId = randomUUID(); + const clientHandshake = RemoteControlClientHandshake.create({ + sessionId: cryptoSessionId, + deviceId: cryptoDeviceId, + commandProfile: "codex", + capabilities: ["workspace.read", "workspace.write", "workspace.exec"], + accountPrivateKey: account.privateKey, + }); + const accepted = acceptRemoteControlClientHello(clientHandshake.hello, { + expectedSessionId: cryptoSessionId, + expectedDeviceId: cryptoDeviceId, + accountPublicKey: account.publicKey, + devicePrivateKey: device.privateKey, + allowedCapabilities: ["workspace.read", "workspace.write", "workspace.exec"], + }); + const clientCipher = clientHandshake.complete(accepted.hello, device.publicKey); + let releaseCommand!: () => void; + let commandStarted!: () => void; + const commandGate = new Promise(resolvePromise => { releaseCommand = resolvePromise; }); + const started = new Promise(resolvePromise => { commandStarted = resolvePromise; }); + const runner: RemoteWorkspaceCommandRunner = { + async run() { + commandStarted(); + // Deliberately ignore AbortSignal: queued operations must still observe their own abort + // after this non-cooperative predecessor finally releases the shared executor queue. + await commandGate; + return { exitCode: 0, stdout: "", stderr: "" }; + }, + }; + + let client: EncryptedRemoteWorkspaceTransport; + let endpoint: EncryptedRemoteWorkspaceExecutorEndpoint; + let responses = 0; + let allResponses!: () => void; + const responsesDone = new Promise(resolvePromise => { allResponses = resolvePromise; }); + client = new EncryptedRemoteWorkspaceTransport({ + executorDeviceId: `device-${cryptoDeviceId}`, + cipher: clientCipher, + // A real WebSocket send settles after queueing bytes, not after remote execution. + sendCiphertext: value => { void endpoint.receiveCiphertext(value); }, + timeoutMs: 100, + }); + endpoint = new EncryptedRemoteWorkspaceExecutorEndpoint({ + executorDeviceId: `device-${cryptoDeviceId}`, + sessionId: cryptoSessionId, + rootId: "project-root", + capabilities: ["workspace.read", "workspace.write", "workspace.exec"], + cipher: accepted.cipher, + executor: new RemoteWorkspaceExecutor({ + deviceId: `device-${cryptoDeviceId}`, + roots: [{ id: "project-root", path: state.executorRoot }], + commandRunner: runner, + }), + sendCiphertext: value => { + client.receiveCiphertext(value); + responses++; + if (responses === 2) allResponses(); + }, + }); + + const first = client.invoke({ + requestId: randomUUID(), sessionId: cryptoSessionId, + executorDeviceId: `device-${cryptoDeviceId}`, rootId: "project-root", + tool: "exec", arguments: { command: ["ignored"], timeoutMs: 60_000 }, + }).catch(error => error as Error); + await started; + const target = join(state.executorRoot, "project", "after-timeout.txt"); + const queued = client.invoke({ + requestId: randomUUID(), sessionId: cryptoSessionId, + executorDeviceId: `device-${cryptoDeviceId}`, rootId: "project-root", + tool: "write_file", + arguments: { path: "project/after-timeout.txt", content: "must-not-run", expectedSha256: null }, + }).catch(error => error as Error); + + const queuedFailure = await queued; + expect(queuedFailure).toBeInstanceOf(Error); + expect((queuedFailure as Error).message).toContain("cancellation was requested"); + expect(existsSync(target)).toBe(false); + releaseCommand(); + const firstFailure = await first; + expect(firstFailure).toBeInstanceOf(Error); + expect((firstFailure as Error).message).toContain("cancellation was requested"); + await responsesDone; + expect(existsSync(target)).toBe(false); + client.close(); + endpoint.close(); + }); }); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 88bd7082674..fdb563d16f1 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -8,6 +8,8 @@ "standalone-build-script.test.ts": "gui", "standalone-service.test.ts": "service", "standalone.test.ts": "lib", + "worker-embed.test.ts": "lib", + "worker-embed-coverage.test.ts": "lib", "server-combo-held-response.test.ts": "server", "key-attribution.test.ts": "usage", "jev-stats.test.ts": "usage", @@ -1654,6 +1656,9 @@ "warmup-registration.test.ts": "ci-workflows", "warmup.test.ts": "codex-integration", "web-search-anthropic.test.ts": "web-search", + "devin-web-search.test.ts": "web-search", + "web-search-recovery-kind.test.ts": "web-search", + "chat-legacy-functions-combo.test.ts": "responses", "web-search-backend-union.test.ts": "web-search", "web-search-bridge-replay.test.ts": "web-search", "web-search-candidates.test.ts": "web-search", diff --git a/tests/images/loop.test.ts b/tests/images/loop.test.ts index 84dd5f13257..f5a06d867ae 100644 --- a/tests/images/loop.test.ts +++ b/tests/images/loop.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { randomUUID } from "node:crypto"; import type { ProviderAdapter, IncomingMeta } from "../../src/adapters/base"; import type { AdapterEvent, OcxParsedRequest } from "../../src/types"; +import type { AttemptRecoveryKind } from "../../src/usage/log"; import type { ImageBridgePlan, ImageCallResult } from "../../src/images/types"; import type { ImageBridgeDeps } from "../../src/images/loop"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; @@ -664,13 +665,16 @@ describe("runWithImageBridge", () => { // First rotation returns a new adapter that also 429s; the exhausted budget must not // re-arm for it. Second call returns null to terminate the pool. return rotations === 1 - ? ({ - ...mockAdapter, - fetchResponse: async () => { - sends += 1; - return new Response("{}", { status: 429 }); - }, - } as ProviderAdapter) + ? { + adapter: { + ...mockAdapter, + fetchResponse: async () => { + sends += 1; + return new Response("{}", { status: 429 }); + }, + } as ProviderAdapter, + recoveryKind: "key-429" as const, + } : null; }, onAttemptSend: recovery => { @@ -927,7 +931,7 @@ describe("runWithImageBridge", () => { retryParsed._kiroAuthContext = { apiRegion: "ap-southeast-2", profileArn: "account-b" }; delete retryParsed._providerContinuation; activeAdapter = secondAdapter; - return secondAdapter; + return { adapter: secondAdapter, recoveryKind: "key-429" }; }, }); const sse = await response.text(); @@ -938,6 +942,50 @@ describe("runWithImageBridge", () => { expect(retryState?._providerContinuation).toBeUndefined(); expect(sse).toContain("after rotate"); }); + + // An account rotation and a key rotation are different operator-facing events, and the + // rotated fetch's recovery kind is the only place the attempt row records which happened. + // The loop used to hardcode `key-429` for both. + test("429 rotation reports the rotator's recovery kind", async () => { + const recoveryKindsFor = async ( + rotation: (next: ProviderAdapter) => { adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind }, + ): Promise<(AttemptRecoveryKind | undefined)[]> => { + let fetchCalls = 0; + const sends: (AttemptRecoveryKind | undefined)[] = []; + const makeAdapter = (label: string): ProviderAdapter => ({ + name: label, + buildRequest: async () => ({ url: "https://test/v1/chat", method: "POST", headers: {}, body: "{}" }), + fetchResponse: async () => { + fetchCalls++; + if (fetchCalls === 1) return new Response("rate limited", { status: 429, headers: { "retry-after": "1" } }); + streamQueue = [[{ type: "text_delta", text: "after rotate" }, { type: "done" }]]; + return new Response("{}", { status: 200 }); + }, + parseStream: async function* (): AsyncGenerator { + const events = streamQueue.shift(); + if (events) for (const e of events) yield e; + }, + }); + const secondAdapter = makeAdapter("after-rotate"); + const response = await runWithImageBridge({ + parsed: makeParsed(), + adapter: makeAdapter("before-rotate"), + plan, + onAttemptSend: recovery => { sends.push(recovery); }, + on429: () => rotation(secondAdapter), + }); + expect(await response.text()).toContain("after rotate"); + return sends; + }; + + // A rotator that crossed accounts says so, and the rotated send carries that kind. + expect(await recoveryKindsFor(next => ({ adapter: next, recoveryKind: "oauth-account-429" }))) + .toEqual([undefined, "oauth-account-429"]); + expect(await recoveryKindsFor(next => ({ adapter: next, recoveryKind: "anthropic-oauth-429" }))) + .toEqual([undefined, "anthropic-oauth-429"]); + expect(await recoveryKindsFor(next => ({ adapter: next, recoveryKind: "key-429" }))) + .toEqual([undefined, "key-429"]); + }); }); // --------------------------------------------------------------------------- diff --git a/tests/lab/lab-fabric-producer-deadline.test.ts b/tests/lab/lab-fabric-producer-deadline.test.ts index f4895b90cfa..5dd73f5b62c 100644 --- a/tests/lab/lab-fabric-producer-deadline.test.ts +++ b/tests/lab/lab-fabric-producer-deadline.test.ts @@ -1,15 +1,17 @@ import { describe, expect, spyOn, test } from "bun:test"; import * as childProcess from "node:child_process"; import { EventEmitter } from "node:events"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { PassThrough } from "node:stream"; import { setImmediate as nextTurn } from "node:timers"; import { runIsolatedFabricProducer } from "../../src/lab/fabric/producer-isolate"; +import { isUnconfirmedProducerTermination } from "../../src/lab/fabric/producer-isolate"; import type { IsolatedProducerResult } from "../../src/lab/fabric/producer-protocol"; import { FabricTaskError, type FabricTaskRunResult, type SyntheticPatchV1 } from "../../src/lab/fabric/types"; import { runFabricSyntheticPatchTaskForRoute } from "../../src/lab/fabric/executor"; +import { createSyntheticScratch } from "../../src/lab/fabric/scratch"; import { createLabDestination } from "../../src/lab/live/destination"; import { fabricCorrectPatchExecutor, fabricMockRoute } from "../helpers/fabric-task-test"; @@ -29,6 +31,8 @@ class DeadlineChild extends EventEmitter { readonly stdout = new PassThrough(); readonly stderr = new PassThrough(); readonly signals: Array = []; + // A successfully spawned child carries a pid; only spawn failures leave it undefined. + readonly pid = 4242; closed = false; kill(signal?: NodeJS.Signals | number): boolean { @@ -36,10 +40,16 @@ class DeadlineChild extends EventEmitter { return true; // Buffered data can arrive after kill; only the test emits close. } - close(): void { + close(code: number | null = 0, signal: NodeJS.Signals | null = null): void { if (this.closed) return; this.closed = true; - this.emit("close", 0, null); + this.emit("close", code, signal); + } + + exit(code: number | null = 0, signal: NodeJS.Signals | null = null): void { + // The process died; `close` is deliberately withheld to model a descendant + // still holding the inherited pipes. + this.emit("exit", code, signal); } } @@ -74,7 +84,7 @@ function installTimers(restorers: Array<() => void>) { type ExpectedFailure = | [code: "inactivity_timeout" | "timeout"] - | [code: "harness_failure", attribution: "harness", message: string]; + | [code: string, attribution: "harness" | "environment", message: string]; type Harness = { child: DeadlineChild; @@ -82,7 +92,9 @@ type Harness = { at: (time: number) => void; result: (newline?: boolean) => void; pending: () => Promise; + outcome: () => Outcome; failure: (...expected: ExpectedFailure) => Promise; + rejection: (...expected: ExpectedFailure) => Promise; success: (lastActivityAt?: number) => Promise; }; @@ -110,23 +122,28 @@ async function withProducer(body: (h: Harness) => Promise, totalTimeoutMs expect(child.stdout.listenerCount("data")).toBe(1); expect(child.listenerCount("close")).toBe(1); expect(timers.map(({ delay }) => delay).sort((a, b) => a - b)).toEqual([IDLE_MS, totalTimeoutMs]); + const rejection = async (...expected: ExpectedFailure) => { + const [code] = expected; + const attribution = expected.length === 3 ? expected[1] : "environment"; + const message = expected.length === 3 ? expected[2] + : code === "inactivity_timeout" ? "inactivity timeout exceeded" : "total timeout exceeded"; + await drain(); + expect(outcome.status).toBe("rejected"); + if (outcome.status !== "rejected") throw new Error("producer did not reject after close"); + expect(outcome.error).toBeInstanceOf(FabricTaskError); + expect(outcome.error).toMatchObject({ code, attribution, message }); + expect(timers.every(({ cleared }) => cleared)).toBe(true); + }; await body({ child, timers, at: (value) => { time = value; }, result: (newline = true) => { child.stdout.write(RESULT + (newline ? "\n" : "")); }, pending: async () => { await drain(); expect(outcome.status).toBe("pending"); }, + outcome: () => outcome, failure: async (...expected) => { - const [code] = expected; - const attribution = code === "harness_failure" ? expected[1] : "environment"; - const message = code === "harness_failure" ? expected[2] - : code === "inactivity_timeout" ? "inactivity timeout exceeded" : "total timeout exceeded"; child.close(); - await drain(); - expect(outcome.status).toBe("rejected"); - if (outcome.status !== "rejected") throw new Error("producer did not reject after close"); - expect(outcome.error).toBeInstanceOf(FabricTaskError); - expect(outcome.error).toMatchObject({ code, attribution, message }); - expect(timers.every(({ cleared }) => cleared)).toBe(true); + await rejection(...expected); }, + rejection, success: async (lastActivityAt = START) => { await drain(); expect(outcome).toEqual({ status: "resolved", value: { patch: PATCH, lastActivityAt } }); @@ -182,7 +199,7 @@ describe("isolated fabric producer deadline admission", () => { h.at(1_101); h.child.stdout.write(ACTIVITY + RESULT + "\n"); await h.pending(); - expect(h.timers).toHaveLength(2); + expect(h.timers).toHaveLength(3); expect(h.child.signals).toEqual(["SIGKILL"]); await h.failure("inactivity_timeout"); }); @@ -250,7 +267,7 @@ describe("isolated fabric producer deadline admission", () => { await h.pending(); expect(h.child.signals).toEqual(["SIGKILL"]); } - expect(h.timers).toHaveLength(2); + expect(h.timers).toHaveLength(3); await h.failure("inactivity_timeout"); }); }); @@ -259,6 +276,8 @@ describe("isolated fabric producer deadline admission", () => { await withProducer(async (h) => { h.at(1_099); h.result(); + await h.pending(); + h.child.close(); await h.success(); }); }); @@ -284,7 +303,7 @@ describe("isolated fabric producer deadline admission", () => { await h.pending(); expect(h.child.signals).toEqual(["SIGKILL"]); } - expect(h.timers).toHaveLength(2); + expect(h.timers).toHaveLength(3); await h.failure("harness_failure", "harness", "first stderr read failure"); }); }); @@ -299,6 +318,8 @@ describe("isolated fabric producer deadline admission", () => { expect(h.timers[1]!.cleared).toBe(false); h.at(1_189); h.result(); + await h.pending(); + h.child.close(); await h.success(1_090); }); }); @@ -312,6 +333,8 @@ describe("isolated fabric producer deadline admission", () => { } h.at(1_249); h.result(); + await h.pending(); + h.child.close(); await h.success(1_180); }); }); @@ -329,6 +352,179 @@ describe("isolated fabric producer deadline admission", () => { }); }); } + + test("protocol error kills the child but settles only at close", async () => { + await withProducer(async (h) => { + h.child.stdout.write('{"type":"error","code":"sandbox_violation","message":"executor reported violation","attribution":"harness"}\n'); + await h.pending(); + expect(h.child.signals).toEqual(["SIGKILL"]); + h.child.close(); + await h.rejection("sandbox_violation", "harness", "executor reported violation"); + }); + }); + + test("stdout stream error kills the child but settles only at close", async () => { + await withProducer(async (h) => { + h.child.stdout.emit("error", new Error("stdout read failure")); + await h.pending(); + expect(h.child.signals).toEqual(["SIGKILL"]); + h.child.close(); + await h.rejection("harness_failure", "harness", "stdout read failure"); + }); + }); + + test("unterminated trailing error at close rejects with its code", async () => { + await withProducer(async (h) => { + h.child.stdout.write('{"type":"error","code":"budget_exhausted","message":"executor spent budget","attribution":"environment"}'); + h.child.close(); + await h.rejection("budget_exhausted", "environment", "executor spent budget"); + }); + }); + + test("a stored result cannot survive a nonzero child exit", async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(); + await h.pending(); + h.child.close(1); + await h.rejection("harness_failure", "harness", "isolated producer exited (1)"); + }); + }); + + test("an unterminated buffered result is rejected on a nonzero exit", async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(false); + await h.pending(); + h.child.close(1); + await h.rejection("harness_failure", "harness", "isolated producer exited (1)"); + }); + }); + + test("a stored result cannot survive a signaled child exit", async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(); + await h.pending(); + h.child.close(null, "SIGKILL"); + await h.rejection("harness_failure", "harness", "isolated producer exited (SIGKILL)"); + }); + }); + + test("a clean exit whose close never arrives is a harness failure", async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(); + h.child.exit(0); + await h.pending(); + expect(h.timers).toHaveLength(3); + h.timers[2]!.callback(); + // The pipes outlived the producer — inconclusive, never a trusted result. + await h.rejection("harness_failure", "harness", "isolated producer exited but its stdio never closed"); + // A descendant may still hold the inherited pipes — and scratch — so the + // rejection must carry the same deferred-cleanup contract as an + // unconfirmed kill, not let the executor remove scratch immediately. + const outcome = h.outcome(); + if (outcome.status !== "rejected") throw new Error("producer did not reject after drain expiry"); + expect(isUnconfirmedProducerTermination(outcome.error)).toBe(true); + expect(h.child.stdout.destroyed).toBe(true); + expect(h.child.stderr.destroyed).toBe(true); + // A close arriving after the decision is ignored. + h.child.close(); + await h.rejection("harness_failure", "harness", "isolated producer exited but its stdio never closed"); + }); + }); + + test("exit disarms the budget timers while close is pending", async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(); + h.child.exit(0); + // The process met its budgets when it died; a deadline must not latch + // while the run waits on a descendant-held pipe to drain. + expect(h.timers[0]!.cleared).toBe(true); + expect(h.timers[1]!.cleared).toBe(true); + await h.pending(); + h.child.close(); + await h.success(); + }); + }); + + test("a buffered result is judged at the exit timestamp, not the close time", async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(false); + h.child.exit(0); + // Clock runs past both deadlines before close admits the drained bytes. + h.at(1_251); + h.child.close(); + await h.success(); + }); + }); + + test("an exit past the deadline is still condemned at the drain", async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(); + h.at(1_251); + h.child.exit(0); + await h.pending(); + expect(h.child.signals).toEqual([]); + h.timers[2]!.callback(); + await h.rejection("inactivity_timeout"); + }); + }); + + test("a latched failure still settles when close never follows exit", async () => { + await withProducer(async (h) => { + h.at(1_100); + h.timers[0]!.callback(); + expect(h.child.signals).toEqual(["SIGKILL"]); + h.child.exit(null, "SIGKILL"); + await h.pending(); + // timers[2] is the kill-confirmation watchdog armed by settleTimeout; + // exit cleared it and armed the drain timer at index 3. + h.timers[3]!.callback(); + await h.rejection("inactivity_timeout"); + }); + }); + + test("a kill that produces no exit or close still rejects within a bound", async () => { + await withProducer(async (h) => { + h.at(1_100); + h.timers[0]!.callback(); + expect(h.child.signals).toEqual(["SIGKILL"]); + // SIGKILL answered by neither exit nor close (uninterruptible child or a + // failed kill): the latched reason must not pend forever. timers[2] is + // the kill-confirmation watchdog armed by settleTimeout. + h.timers[2]!.callback(); + await h.rejection("inactivity_timeout"); + const outcome = h.outcome(); + expect(outcome.status).toBe("rejected"); + if (outcome.status === "rejected") { + expect(isUnconfirmedProducerTermination(outcome.error)).toBe(true); + } + // A close arriving after the bounded rejection is ignored. + h.child.close(); + await h.rejection("inactivity_timeout"); + }); + }); + + test("a nonzero exit without close still rejects after drain", async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(); + h.child.exit(1); + await h.pending(); + h.timers[2]!.callback(); + await h.rejection("harness_failure", "harness", "isolated producer exited (1); its stdio never closed"); + // The pipes outlived the producer here too, so the rejection carries + // the same deferred-cleanup contract as a clean exit without close. + const outcome = h.outcome(); + if (outcome.status !== "rejected") throw new Error("producer did not reject after drain expiry"); + expect(isUnconfirmedProducerTermination(outcome.error)).toBe(true); + }); + }); }); test("trusted route keeps scratch until stderr-failed child closes, then cleans it", async () => { @@ -421,3 +617,99 @@ test("trusted route keeps scratch until stderr-failed child closes, then cleans } } }); + +for (const marker of ["absent", "regular", "symlink", "directory"] as const) { +test.skipIf(process.platform === "win32" && marker === "symlink")(`unconfirmed producer retains scratch and preserves ${marker} marker paths`, async () => { + const configDir = mkdtempSync(join(tmpdir(), "ocx-fabric-consumer-defer-")); + const child = new DeadlineChild(); + Object.assign(child.stdout, { unref() {} }); + Object.assign(child.stderr, { unref() {} }); + const originals = { spawn: childProcess.spawn, set: globalThis.setTimeout, clear: globalThis.clearTimeout }; + const restorers: Array<() => void> = []; + const proxyNames = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "all_proxy", "no_proxy"]; + const proxyEnv = proxyNames.map((name) => [name, process.env[name]] as const); + const outer: { outcome: Outcome } = { outcome: { status: "pending" } }; + try { + for (const name of proxyNames) delete process.env[name]; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", labRunApproval: true, configDir, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + }); + const spawnSpy = spyOn(childProcess, "spawn").mockImplementation(() => child as unknown as childProcess.ChildProcess); + restorers.push(() => spawnSpy.mockRestore()); + const timers = installTimers(restorers); + void runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), destination, configDir, now: () => START, + patchExecutor: fabricCorrectPatchExecutor(), + }).then( + (value) => { outer.outcome = { status: "resolved", value }; }, + (error: unknown) => { outer.outcome = { status: "rejected", error }; }, + ); + const scratchRoot = spawnSpy.mock.calls[0]?.[2]?.env?.OCX_FABRIC_SCRATCH_ROOT; + expect(typeof scratchRoot).toBe("string"); + if (!scratchRoot) throw new Error("producer spawn omitted its scratch root"); + expect(existsSync(scratchRoot)).toBe(true); + await drain(); + + const markerPath = join(scratchRoot, ".ocx-deferred-cleanup"); + const external = join(configDir, "outside.txt"); + writeFileSync(external, "preserve outside bytes\n"); + if (marker === "regular") { + writeFileSync(markerPath, "producer-owned marker\n"); + utimesSync(markerPath, 0, 0); + } else if (marker === "symlink") { + symlinkSync(external, markerPath); + } else if (marker === "directory") { + mkdirSync(markerPath); + } + + // The producer exits cleanly but a descendant keeps the inherited pipes + // open, so close never arrives and the drain bound expires. + child.exit(0); + await drain(); + timers[2]!.callback(); + await drain(); + expect(outer.outcome.status).toBe("resolved"); + if (outer.outcome.status !== "resolved") throw new Error("route did not settle after drain expiry"); + expect(outer.outcome.value).toMatchObject({ + outcome: { + outcome: "inconclusive", + failure: { class: "harness_failure", code: "harness_failure", attribution: "harness" }, + }, + }); + + expect(existsSync(scratchRoot)).toBe(true); + expect(readFileSync(external, "utf8")).toBe("preserve outside bytes\n"); + if (marker === "absent") expect(existsSync(markerPath)).toBe(false); + if (marker === "symlink") expect(lstatSync(markerPath).isSymbolicLink()).toBe(true); + if (marker === "regular") expect(readFileSync(markerPath, "utf8")).toBe("producer-owned marker\n"); + // A later task cannot use even an epoch-old marker as permission to delete. + const next = createSyntheticScratch(configDir); + next.cleanup(); + expect(existsSync(scratchRoot)).toBe(true); + // Closing inherited pipes does not establish that the writer has exited. + child.close(); + child.stdout.destroy(); child.stderr.destroy(); + await drain(); + expect(existsSync(scratchRoot)).toBe(true); + expect(readFileSync(join(scratchRoot, "src/value.txt"), "utf8")).toBe("before\n"); + } finally { + try { + child.close(); + await drain(); + child.stdin.destroy(); child.stdout.destroy(); child.stderr.destroy(); + } finally { + for (const restore of restorers.reverse()) restore(); + for (const [name, value] of proxyEnv) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + rmSync(configDir, { recursive: true, force: true }); + expect(childProcess.spawn).toBe(originals.spawn); + expect(globalThis.setTimeout).toBe(originals.set); + expect(globalThis.clearTimeout).toBe(originals.clear); + } + } +}); + +} diff --git a/tests/lab/lab-fabric-task.test.ts b/tests/lab/lab-fabric-task.test.ts index a06087eeceb..4ee3acaa5b0 100644 --- a/tests/lab/lab-fabric-task.test.ts +++ b/tests/lab/lab-fabric-task.test.ts @@ -184,6 +184,120 @@ export async function execute(_input: FabricPatchExecutorInput): Promise { + process.stdout.write(JSON.stringify({ type: "result", patch }) + "\\n"); + const deadline = Date.now() + ${FAST_FABRIC_ISOLATION.totalTimeoutMs + 500}; + while (Date.now() < deadline) { + input.reportActivity(); + await Bun.sleep(100); + } + writeFileSync(${JSON.stringify(marker)}, "late\\n"); + return patch; +} +`); + return { + executor: createHostIssuedFabricPatchExecutor(modulePath, async () => correctSyntheticPatch()), + marker, + }; +} + +function fabricEarlyErrorPatchExecutor(home: string): { executor: TrustedFabricPatchExecutor; marker: string } { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "early-error-patch.ts"); + const marker = join(home, "late-child-error-mutation.txt"); + writeFileSync(modulePath, ` +import { writeFileSync } from "node:fs"; +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +const patch: SyntheticPatchV1 = { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], +}; + +export async function execute(input: FabricPatchExecutorInput): Promise { + process.stdout.write(JSON.stringify({ type: "error", code: "harness_failure", message: "executor reported failure", attribution: "harness" }) + "\\n"); + const deadline = Date.now() + ${FAST_FABRIC_ISOLATION.totalTimeoutMs + 500}; + while (Date.now() < deadline) { + input.reportActivity(); + await Bun.sleep(100); + } + writeFileSync(${JSON.stringify(marker)}, "late\\n"); + return patch; +} +`); + return { + executor: createHostIssuedFabricPatchExecutor(modulePath, async () => correctSyntheticPatch()), + marker, + }; +} + +function fabricResultThenExitPatchExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "result-then-exit-patch.ts"); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +const patch: SyntheticPatchV1 = { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], +}; + +export function execute(_input: FabricPatchExecutorInput): Promise { + process.stdout.write(JSON.stringify({ type: "result", patch }) + "\\n", () => process.exit(1)); + return new Promise(() => {}); +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => correctSyntheticPatch()); +} + +function fabricOrphanedPipePatchExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "orphaned-pipe-patch.ts"); + writeFileSync(modulePath, ` +import { spawn } from "node:child_process"; +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +const patch: SyntheticPatchV1 = { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], +}; + +export function execute(_input: FabricPatchExecutorInput): Promise { + // A detached descendant holds the inherited stdio open after this process + // exits, so the parent's "close" event cannot arrive on its own. + const descendant = spawn(process.execPath, ["-e", "setTimeout(() => {}, 4000)"], { + stdio: ["ignore", "inherit", "inherit"], + detached: true, + }); + descendant.unref(); + process.stdout.write(JSON.stringify({ type: "result", patch }) + "\\n", () => process.exit(0)); + return new Promise(() => {}); +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => correctSyntheticPatch()); +} + function fabricTraversalPatchExecutor(home: string): TrustedFabricPatchExecutor { const dir = join(home, "fabric-executors"); mkdirSync(dir, { recursive: true }); @@ -711,6 +825,74 @@ export async function execute() { expect(result.outcome.failure?.code).toBe("inactivity_timeout"); }, 20_000); + test("producer result remains supervised until the child exits", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const { executor, marker } = fabricEarlyResultPatchExecutor(home); + const startedAt = Date.now(); + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: executor, + configDir: home, + }); + expect(result.outcome.outcome).not.toBe("pass"); + expect(result.outcome.failure?.code).toBe("timeout"); + // The fixture's late write lands totalTimeoutMs + 500 after the child starts. + // The task settles near that deadline, so only the remainder must elapse — + // a fixed totalTimeoutMs sleep would overflow the test timeout under CI scaling. + await Bun.sleep(Math.max(1_000, FAST_FABRIC_ISOLATION.totalTimeoutMs + 750 - (Date.now() - startedAt))); + expect(existsSync(marker)).toBe(false); + }, 45_000); + + test("producer error remains supervised until the child exits", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const { executor, marker } = fabricEarlyErrorPatchExecutor(home); + const startedAt = Date.now(); + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: executor, + configDir: home, + }); + expect(result.outcome.outcome).not.toBe("pass"); + expect(result.outcome.failure?.code).toBe("harness_failure"); + // Same bounded wait as above: the child settles fast, so the marker deadline + // is still roughly a full budget away under CI-scaled isolation limits. + await Bun.sleep(Math.max(1_000, FAST_FABRIC_ISOLATION.totalTimeoutMs + 750 - (Date.now() - startedAt))); + expect(existsSync(marker)).toBe(false); + }, 45_000); + + test("producer result is rejected when the child exits nonzero", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricResultThenExitPatchExecutor(home), + configDir: home, + }); + expect(result.outcome.outcome).not.toBe("pass"); + expect(result.outcome.failure?.code).toBe("harness_failure"); + }, 20_000); + + test("producer result is rejected when a descendant holds the pipes", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricOrphanedPipePatchExecutor(home), + configDir: home, + }); + // `close` never follows `exit` while a descendant holds the pipes — the + // process tree may have escaped supervision, so the result is rejected as + // an inconclusive harness failure rather than trusted. + expect(result.outcome.outcome).not.toBe("pass"); + expect(result.outcome.failure?.code).toBe("harness_failure"); + }, 20_000); + test("activity resets inactivity deadline within total budget", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; diff --git a/tests/lib/worker-embed-coverage.test.ts b/tests/lib/worker-embed-coverage.test.ts new file mode 100644 index 00000000000..37b4256d4f6 --- /dev/null +++ b/tests/lib/worker-embed-coverage.test.ts @@ -0,0 +1,66 @@ +/** + * Every worker a compiled binary can start must be embedded by build-standalone. + * + * `spawnWorker(url, key)` looks `key` up in the bundles scripts/build-standalone.ts generates and + * falls back to `url` when it is missing. The fallback is right for source checkouts and wrong + * in a compiled binary, where the nested entrypoint does not resolve (oven-sh/bun#29124). A key + * that drifts from `WORKER_ENTRIES`, or a new bare `new Worker(new URL(...))`, therefore passes + * every source-run test and breaks only the released binary — the failure #5761 fixed. + */ +import { expect, test } from "bun:test"; +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import { repoPath } from "../helpers/repo-root"; + +// Bare workers that a compiled binary can never start, each with the reason. +const BARE_WORKER_EXEMPTIONS: Record = { + // Only reached when the installer is pnpm, which runs the package from source, not the binary. + "src/update/async-check.ts": "pnpm-only owner lookup", +}; + +function sourceFiles(dir: string): string[] { + return readdirSync(dir).flatMap(name => { + const path = join(dir, name); + if (statSync(path).isDirectory()) return name === "generated" ? [] : sourceFiles(path); + return /\.(?:ts|mts|mjs)$/.test(name) ? [path] : []; + }); +} + +function workerEntries(): Map { + const script = readFileSync(repoPath("scripts", "build-standalone.ts"), "utf8"); + const block = /const WORKER_ENTRIES[^{]*\{([\s\S]*?)\n\};/.exec(script)?.[1]; + if (!block) throw new Error("WORKER_ENTRIES not found in scripts/build-standalone.ts"); + const entries = new Map(); + for (const match of block.matchAll(/"([^"]+)":\s*join\(repoRoot,\s*([^)]*)\)/g)) { + const segments = [...match[2]!.matchAll(/"([^"]+)"/g)].map(segment => segment[1]!); + entries.set(match[1]!, segments.join("/")); + } + return entries; +} + +const root = repoPath(); +const files = sourceFiles(repoPath("src")).map(path => ({ + rel: relative(root, path).split("\\").join("/"), + text: readFileSync(path, "utf8"), +})); + +test("every spawnWorker key is embedded for the same worker file", () => { + const entries = workerEntries(); + const spawned = new Map(); + for (const { rel, text } of files) { + for (const match of text.matchAll(/spawnWorker\(\s*new URL\("\.\/([^"]+)",\s*import\.meta\.url\)\.href,\s*"([^"]+)"\s*\)/g)) { + spawned.set(match[2]!, `${rel.slice(0, rel.lastIndexOf("/"))}/${match[1]}`); + } + } + expect(spawned.size).toBeGreaterThan(0); + for (const [key, workerFile] of spawned) expect({ key, embedded: entries.get(key) }).toEqual({ key, embedded: workerFile }); + // A stale entry bundles a worker nobody spawns, and usually means a key was renamed on one side. + expect([...entries.keys()].filter(key => !spawned.has(key))).toEqual([]); +}); + +test("workers are started through spawnWorker unless explicitly exempted", () => { + const bare = files + .filter(({ rel, text }) => rel !== "src/lib/worker-embed.ts" && /new Worker\(/.test(text.replace(/^\s*(?:\*|\/\/).*$/gm, ""))) + .map(({ rel }) => rel); + expect(bare.sort()).toEqual(Object.keys(BARE_WORKER_EXEMPTIONS).sort()); +}); diff --git a/tests/lib/worker-embed.test.ts b/tests/lib/worker-embed.test.ts new file mode 100644 index 00000000000..024322d7933 --- /dev/null +++ b/tests/lib/worker-embed.test.ts @@ -0,0 +1,66 @@ +/** + * Regression for standalone-binary worker spawning. + * + * `bun build --compile` cannot resolve nested worker entrypoints from + * /$bunfs (oven-sh/bun#29124), so standalone builds pre-bundle each worker + * source and `spawnWorker` starts it from a Blob URL instead. Source + * checkouts keep the original `new Worker(url)` fallback. These cases pin + * both branches so a refactor cannot silently revert to bare + * `new Worker(new URL(...))` call sites, which die with + * `ModuleNotFound resolving "/$bunfs/root/.ts" (entry point)` in + * compiled binaries. + */ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { WORKER_BUNDLES } from "../../src/generated/worker-bundles.gen"; +import { spawnWorker } from "../../src/lib/worker-embed"; + +function nextMessage(worker: Worker, timeoutMs = 5000): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("worker message timeout")), timeoutMs); + worker.onmessage = (event) => { + clearTimeout(timer); + resolve(event.data); + }; + worker.onerror = (event) => { + clearTimeout(timer); + reject(new Error(`worker error: ${String(event.message ?? event)}`)); + }; + }); +} + +afterEach(() => { + for (const key of Object.keys(WORKER_BUNDLES)) delete WORKER_BUNDLES[key]; +}); + +describe("spawnWorker", () => { + test("spawns from the embedded bundle via Blob URL when one is registered", async () => { + // The devUrl must be ignored when a bundle is registered; "missing:" is + // not a loadable URL, so reaching the message proves the Blob path ran. + WORKER_BUNDLES["test-echo"] = `postMessage("embedded-ok");`; + const worker = spawnWorker("missing://no-such-url", "test-echo"); + try { + await expect(nextMessage(worker)).resolves.toBe("embedded-ok"); + } finally { + worker.terminate(); + } + }); + + test("falls back to the dev URL when no bundle is registered", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-worker-embed-")); + try { + writeFileSync(join(dir, "dev-worker.js"), `postMessage("dev-ok");`); + const worker = spawnWorker(pathToFileURL(join(dir, "dev-worker.js")).href, "not-registered"); + try { + await expect(nextMessage(worker)).resolves.toBe("dev-ok"); + } finally { + worker.terminate(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/oauth/generic-oauth-failover.test.ts b/tests/oauth/generic-oauth-failover.test.ts index 342b293e739..2260a8248ea 100644 --- a/tests/oauth/generic-oauth-failover.test.ts +++ b/tests/oauth/generic-oauth-failover.test.ts @@ -278,6 +278,17 @@ describe("#2568 generic OAuth account failover", () => { expect(genericFailoverRetryAfterSeconds("xai")).toBeGreaterThan(500); }); + test("a stated Retry-After past the local cap is honoured, not truncated", async () => { + const ids = await seed(2); + const now = Date.now(); + // One hour. Retrying before it elapses buys a second 429 on an account upstream already + // told us to leave alone — the exact wasted send this pool exists to avoid. + expect(rotateGenericOAuthAccountOn429(config(), "xai", ids[0]!, "3600", now)).toBe(ids[1]); + expect(genericFailoverRetryAfterSeconds("xai", now)).toBe(3600); + // Still cooled long after any local truncation would have released it. + expect(eligibleFailoverAccounts("xai", now + 20 * 60_000)).toEqual([ids[1]!]); + }); + test("an excluded provider is never enabled, however many accounts it has", async () => { await seed(2); // Codex and Anthropic own quota scopes, probe leases and affinity that this must not diff --git a/tests/providers/kiro/kiro-review-regressions.test.ts b/tests/providers/kiro/kiro-review-regressions.test.ts index 70d0760eb59..0fd805ea86f 100644 --- a/tests/providers/kiro/kiro-review-regressions.test.ts +++ b/tests/providers/kiro/kiro-review-regressions.test.ts @@ -11,7 +11,7 @@ import { readKiroCliSqliteCredential, restoreStaleKiroCliSessionRecovery, } from "../../../src/oauth/kiro-credentials"; -import { getAccountCredential, getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; +import { getAccountCredential, getAccountSet, saveCredential, saveCredentialWithReceipt, setActiveAccount } from "../../../src/oauth/store"; import type { OAuthController, OAuthCredentials } from "../../../src/oauth/types"; import type { OcxConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; @@ -253,6 +253,98 @@ describe("Kiro review regressions", () => { }); }); + test("a failed forced login removes only its own account and preserves a concurrent login", async () => { + const accountA = "arn:aws:codewhisperer:us-east-1:123456789012:profile/a"; + const accountB = "arn:aws:codewhisperer:us-east-1:123456789012:profile/b"; + const accountC = "arn:aws:codewhisperer:us-east-1:123456789012:profile/c"; + await saveCredential("kiro", { + access: "access-a", refresh: "refresh-a", expires: Date.now() + 60_000, accountId: accountA, + }); + const rawCredential: OAuthCredentials = { + access: "access-c", refresh: "refresh-c", expires: Date.now() + 60_000, accountId: accountC, + }; + const originalLogin = OAUTH_PROVIDERS.kiro.login; + OAUTH_PROVIDERS.kiro.login = async () => rawCredential; + try { + await expect(runLogin("kiro", {} as OAuthController, { forceLogin: true }, { + loadConfig: config, + saveCredentialWithReceipt: async (provider, credential, options) => { + const receipt = await saveCredentialWithReceipt(provider, credential, options); + await saveCredential("kiro", { + access: "access-b", refresh: "refresh-b", expires: Date.now() + 60_000, accountId: accountB, + }, { preserveIdentityless: true }); + return receipt; + }, + saveConfig: () => { throw new Error("config publication failed"); }, + settleKiroLoginTransaction: () => {}, + })).rejects.toThrow("config publication failed"); + } finally { + OAUTH_PROVIDERS.kiro.login = originalLogin; + } + + const set = getAccountSet("kiro")!; + expect(set.accounts.map(account => account.credential.accountId).sort()).toEqual([accountA, accountB]); + expect(getAccountCredential("kiro", set.activeAccountId)?.accountId).toBe(accountB); + }); + + test("rollback leaves a concurrently refreshed copy of the same account untouched", async () => { + const accountA = "arn:aws:codewhisperer:us-east-1:123456789012:profile/base"; + const accountC = "arn:aws:codewhisperer:us-east-1:123456789012:profile/owned"; + await saveCredential("kiro", { + access: "access-base", refresh: "refresh-base", expires: Date.now() + 60_000, accountId: accountA, + }); + const rawCredential: OAuthCredentials = { + access: "access-owned", refresh: "refresh-owned", expires: Date.now() + 60_000, accountId: accountC, + }; + const originalLogin = OAUTH_PROVIDERS.kiro.login; + OAUTH_PROVIDERS.kiro.login = async () => rawCredential; + try { + await expect(runLogin("kiro", {} as OAuthController, { forceLogin: true }, { + loadConfig: config, + saveCredentialWithReceipt: async (provider, credential, options) => { + const receipt = await saveCredentialWithReceipt(provider, credential, options); + await saveCredential("kiro", { + ...rawCredential, + access: "access-concurrent", + refresh: "refresh-concurrent", + }, { preserveIdentityless: true }); + return receipt; + }, + saveConfig: () => { throw new Error("config publication failed"); }, + settleKiroLoginTransaction: () => {}, + })).rejects.toThrow("config publication failed"); + } finally { + OAUTH_PROVIDERS.kiro.login = originalLogin; + } + + const set = getAccountSet("kiro")!; + const concurrent = set.accounts.find(account => account.credential.accountId === accountC)!; + expect(concurrent.credential).toMatchObject({ + access: "access-concurrent", + refresh: "refresh-concurrent", + }); + expect(set.activeAccountId).toBe(concurrent.id); + }); + + test("a failed first forced login rolls back the newly created provider set", async () => { + const rawCredential: OAuthCredentials = { + access: "access-first", refresh: "refresh-first", expires: Date.now() + 60_000, + accountId: "arn:aws:codewhisperer:us-east-1:123456789012:profile/first-only", + }; + const originalLogin = OAUTH_PROVIDERS.kiro.login; + OAUTH_PROVIDERS.kiro.login = async () => rawCredential; + try { + await expect(runLogin("kiro", {} as OAuthController, { forceLogin: true }, { + loadConfig: config, + saveConfig: () => { throw new Error("config publication failed"); }, + settleKiroLoginTransaction: () => {}, + })).rejects.toThrow("config publication failed"); + } finally { + OAUTH_PROVIDERS.kiro.login = originalLogin; + } + expect(getAccountSet("kiro")).toBeNull(); + }); + test("forced login refuses custom import DB selectors that diverge from the CLI store", async () => { seedKiroCliDb("aoa-primary", "rt-primary", { profileArn: "arn:aws:codewhisperer:us-east-1:123456789012:profile/primary", diff --git a/tests/responses/chat-completions-endpoint.test.ts b/tests/responses/chat-completions-endpoint.test.ts index d292da0918b..1913cd76177 100644 --- a/tests/responses/chat-completions-endpoint.test.ts +++ b/tests/responses/chat-completions-endpoint.test.ts @@ -2553,7 +2553,9 @@ test("collectChatCompletion accounts split surrogate content incrementally and r expect(await collectChatCompletion(stream, "mock/test-model", budget)).toMatchObject({ choices: [{ message: { content: "😀", reasoning_content: "😀", refusal: "😀" } }], }); - expect(budget.snapshot().currentBytes).toBe(12); + // Three retained fields each contain one completed scalar. This must match the runtime's + // full-string UTF-8 sizing even when its isolated-surrogate sizing differs. + expect(budget.snapshot().currentBytes).toBe(3 * Buffer.byteLength("😀")); } } finally { budget.dispose(); } } diff --git a/tests/responses/chat-legacy-functions-combo.test.ts b/tests/responses/chat-legacy-functions-combo.test.ts new file mode 100644 index 00000000000..2561e4d6e72 --- /dev/null +++ b/tests/responses/chat-legacy-functions-combo.test.ts @@ -0,0 +1,137 @@ +/** + * Legacy Chat `functions` history across a reasoning-preserving combo failover. + * + * #5844 translates legacy `functions` / `function_call` / `role: "function"` into Responses + * tools and a paired call/output, and #5843 keeps the Chat reasoning intent through a combo + * whose first target has an empty effort ladder. Each is pinned on its own; this case pins the + * union on one request, read off the wire of both loopback upstreams. + */ +import { afterEach, beforeEach, expect, setDefaultTimeout, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleChatCompletions } from "../../src/server/chat-completions"; +import { clearComboSelectionState, clearComboTargetCooldowns } from "../../src/combos"; +import { clearComboRecallForTests } from "../../src/server/responses/combo-session-recall"; +import { clearKeyCooldowns } from "../../src/providers/key-failover"; +import { clearResponseStateForTests, flushResponseState } from "../../src/responses/state"; +import { resetProviderRequestPacingForTest } from "../../src/providers/request-pacing"; +import { responsesSuccess } from "../helpers/combo-failover-upstream"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; + +type Rec = Record; + +setDefaultTimeout(30_000); + +let testDir = ""; +let previousHome: string | undefined; +let isolatedCodexHome: IsolatedCodexHome | null = null; +let releaseSpendHome: (() => void) | undefined; +const servers: Array> = []; + +function resetRoutingState(): void { + clearComboSelectionState(); + clearComboRecallForTests(); + clearComboTargetCooldowns(); + clearKeyCooldowns(); +} + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + isolatedCodexHome = installIsolatedCodexHome("ocx-chat-legacy-combo-codex-"); + testDir = mkdtempSync(join(tmpdir(), "ocx-chat-legacy-combo-")); + process.env.OPENCODEX_HOME = testDir; + releaseSpendHome = acquireOwnedSpendHome(); + resetRoutingState(); + clearResponseStateForTests(); +}); + +afterEach(async () => { + releaseSpendHome?.(); + releaseSpendHome = undefined; + for (const server of servers.splice(0)) await server.stop(true); + await flushResponseState(); + clearResponseStateForTests(); + resetRoutingState(); + resetProviderRequestPacingForTest(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + if (testDir) removeTreeWithRetry(testDir); +}); + +function upstream(answer: () => Response) { + const bodies: Rec[] = []; + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + bodies.push(await request.json() as Rec); + return answer(); + }, + }); + servers.push(server); + return { bodies, baseUrl: new URL("/v1", server.url).href }; +} + +function provider(baseUrl: string, extra: Partial): OcxProviderConfig { + return { adapter: "openai-responses", baseUrl, allowPrivateNetwork: true, authMode: "key", apiKey: "key", ...extra }; +} + +function responsesStream(text: string): Response { + return new Response([ + `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", delta: text, item_id: "msg_backup", output_index: 0, content_index: 0 })}\n\n`, + `event: response.completed\ndata: ${JSON.stringify({ type: "response.completed", response: responsesSuccess(text, "m2") })}\n\n`, + ].join(""), { headers: { "content-type": "text/event-stream" } }); +} + +test("legacy function history keeps its pairing and reasoning intent through combo failover", async () => { + const a = upstream(() => Response.json({ error: { message: "fixture outage" } }, { status: 503 })); + const b = upstream(() => responsesStream("sunny")); + const config: OcxConfig = { + port: 0, + defaultProvider: "a", + providers: { + a: provider(a.baseUrl, { reasoningEfforts: [] }), + b: provider(b.baseUrl, { reasoningEfforts: ["low", "high"] }), + }, + combos: { pair: { strategy: "failover", targets: [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }] } }, + }; + + const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "combo/pair", + stream: false, + reasoning_effort: "high", + messages: [ + { role: "user", content: "weather in Seoul?" }, + { role: "assistant", content: null, function_call: { name: "get_weather", arguments: "{\"city\":\"Seoul\"}" } }, + { role: "function", name: "get_weather", content: "{\"temp\":21}" }, + ], + functions: [{ name: "get_weather", parameters: { type: "object", properties: { city: { type: "string" } } } }], + function_call: "auto", + }), + }), config, { model: "", provider: "" }, { requestId: `legacy-combo-${crypto.randomUUID()}`, start: Date.now() }); + + expect(response.status).toBe(200); + expect(await response.text()).toContain("sunny"); + expect(a.bodies.length).toBeGreaterThan(0); + for (const body of a.bodies) expect((body.reasoning as Rec | undefined)?.effort).toBeUndefined(); + + expect(b.bodies).toHaveLength(1); + const sent = b.bodies[0]!; + expect((sent.reasoning as Rec | undefined)?.effort).toBe("high"); + expect(sent.tool_choice).toBe("auto"); + expect(sent.tools).toEqual([expect.objectContaining({ type: "function", name: "get_weather" })]); + const input = sent.input as Rec[]; + const call = input.find(item => item.type === "function_call"); + const output = input.find(item => item.type === "function_call_output"); + expect(call).toMatchObject({ name: "get_weather", arguments: "{\"city\":\"Seoul\"}" }); + expect(output).toMatchObject({ call_id: call!.call_id, output: "{\"temp\":21}" }); +}); diff --git a/tests/responses/chat-media-translation.test.ts b/tests/responses/chat-media-translation.test.ts index d37c27ec871..06406b272b3 100644 --- a/tests/responses/chat-media-translation.test.ts +++ b/tests/responses/chat-media-translation.test.ts @@ -74,6 +74,56 @@ describe("Chat media stays native or fails explicitly at translation", () => { expect(() => chatCompletionsToResponsesBody(raw)).toThrow("Legacy function-result image translation is not implemented"); }); + test("legacy declarations, calls and textual results translate as one paired tool exchange", () => { + const translated = chatCompletionsToResponsesBody({ + model: "model", + functions: [{ name: "lookup", description: "Look up a value", parameters: { + type: "object", properties: { key: { type: "string" } }, required: ["key"], + } }], + function_call: { name: "lookup" }, + messages: [ + { role: "user", content: "Find it." }, + { role: "assistant", content: null, function_call: { name: "lookup", arguments: '{"key":"answer"}' } }, + { role: "function", name: "lookup", content: "RESULT_42" }, + { role: "assistant", content: "The result is 42." }, + ], + }); + + expect(translated.tools).toEqual([{ + type: "function", name: "lookup", description: "Look up a value", + parameters: { type: "object", properties: { key: { type: "string" } }, required: ["key"] }, + }]); + expect(translated.tool_choice).toEqual({ type: "function", name: "lookup" }); + const input = translated.input as Array>; + const call = input.find(item => item.type === "function_call")!; + const output = input.find(item => item.type === "function_call_output")!; + expect(call).toMatchObject({ name: "lookup", arguments: '{"key":"answer"}' }); + expect(output).toEqual({ type: "function_call_output", call_id: call.call_id, output: "RESULT_42" }); + expect(input).toContainEqual({ + type: "message", role: "assistant", + content: [{ type: "output_text", text: "The result is 42." }], + }); + }); + + test("a null legacy function call preserves a textual assistant message", () => { + const translated = chatCompletionsToResponsesBody({ + model: "model", + messages: [{ role: "assistant", content: "The result is 42.", function_call: null }], + }); + + expect(translated.input).toEqual([{ + type: "message", role: "assistant", + content: [{ type: "output_text", text: "The result is 42." }], + }]); + }); + + test("an orphan legacy function result is rejected instead of silently discarded", () => { + expect(() => chatCompletionsToResponsesBody({ + model: "model", + messages: [{ role: "user", content: "go" }, { role: "function", name: "lookup", content: "orphan" }], + })).toThrow("function result has no pending call named lookup"); + }); + test("plain text mentioning an attachment is not treated as one", () => { const text = JSON.stringify([...media, INLINE_FILE]); const out = chatCompletionsToResponsesBody({ model: "model", messages: [{ role: "user", content: text }] }); diff --git a/tests/responses/chat-native-combo.test.ts b/tests/responses/chat-native-combo.test.ts index a979edd1cb8..e4bda20b6af 100644 --- a/tests/responses/chat-native-combo.test.ts +++ b/tests/responses/chat-native-combo.test.ts @@ -207,6 +207,60 @@ describe("native Chat candidates in a combo", () => { }); }); + test("Chat reasoning intent survives an empty-ladder first target and reaches failover", async () => { + const a = upstream(() => Response.json({ error: { message: "fixture outage" } }, { status: 503 })); + const b = upstream(() => responsesStream("reasoned fallback")); + const config = comboConfig( + { + a: provider("openai-responses", a.baseUrl, { reasoningEfforts: [] }), + b: provider("openai-responses", b.baseUrl, { reasoningEfforts: ["low", "high"] }), + }, + [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }], + ); + + const { response, text } = await send(config, { stream: false, reasoning_effort: "high" }); + + expect(response.status).toBe(200); + expect(text).toContain("reasoned fallback"); + expect(a.bodies).toHaveLength(3); + for (const body of a.bodies) { + expect((body.reasoning as Rec | undefined)?.effort).toBeUndefined(); + } + expect(b.bodies).toHaveLength(1); + expect((b.bodies[0]!.reasoning as Rec | undefined)?.effort).toBe("high"); + }); + + test("Chat policy fallback strips only the empty-ladder attempt's reasoning effort", async () => { + const a = upstream(() => Response.json({ error: { message: "fixture outage" } }, { status: 503 })); + const b = upstream(() => responsesStream("reasoned policy fallback")); + const config: OcxConfig = { + port: 0, + defaultProvider: "a", + providers: { + // Anthropic consumes parsed options rather than the Responses raw-body + // sanitizer, so this catches an effort that leaks past policy selection. + a: provider("anthropic", a.baseUrl, { models: ["m1"], reasoningEfforts: [] }), + b: provider("openai-responses", b.baseUrl, { models: ["m2"], reasoningEfforts: ["low", "high"] }), + }, + routingProfiles: { daily: { candidates: [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }] } }, + }; + + const { response, text, rows } = await send(config, { + model: "policy/daily", stream: false, reasoning_effort: "high", include_reasoning: true, + }); + + expect(response.status).toBe(200); + expect(text).toContain("reasoned policy fallback"); + expect(a.bodies.length).toBeGreaterThan(0); + for (const body of a.bodies) { + expect(body.thinking).toBeUndefined(); + expect(body.output_config).toBeUndefined(); + } + expect(b.bodies).toHaveLength(1); + expect(b.bodies[0]!.reasoning).toMatchObject({ effort: "high", summary: "auto" }); + expect(rows[0]!.attempts?.map(attempt => attempt.status)).toEqual([503, 200]); + }); + test("a streamed native answer that fails after output is not re-sent to the next target", async () => { const a = upstream(() => chatErrorStream("fixture broke mid-stream", "partial answer")); const b = upstream(() => responsesStream("must not run")); diff --git a/tests/responses/reasoning-effort-summary-default.test.ts b/tests/responses/reasoning-effort-summary-default.test.ts index d00d46178e6..564af8427ef 100644 --- a/tests/responses/reasoning-effort-summary-default.test.ts +++ b/tests/responses/reasoning-effort-summary-default.test.ts @@ -110,4 +110,33 @@ describe("reasoning effort preserves visible thinking when summary is omitted", const child = concreteComboRequestBody(body, target, "high", ["high"]); expect(child.reasoning).toEqual({ effort: "high", summary: "auto" }); }); + + test("Chat final-route normalization strips both shapes without losing summary or later-route effort", async () => { + const config: OcxConfig = { + port: 0, + defaultProvider: "a", + providers: { + a: { adapter: "anthropic", baseUrl: "https://a.example.test/v1", reasoningEfforts: [] }, + b: { adapter: "anthropic", baseUrl: "https://b.example.test/v1", reasoningEfforts: ["low", "high"] }, + }, + }; + for (const [provider, expectedEffort] of [["a", undefined], ["b", "high"]] as const) { + // Policy fallback parses each attempt from its preserved wire snapshot. Keep these + // cases independent so the empty-ladder mutation cannot manufacture the capable result. + const parsed = parseRequest({ + model: `${provider}/test-model`, input: [], reasoning: { effort: "high", summary: "auto" }, + }); + const route = routeModel(config, `${provider}/test-model`); + await applyFinalRouteRequestNormalization({ + parsed, route, config, + req: new Request("http://localhost/v1/responses"), + logCtx: { model: parsed.modelId, provider }, + inboundWire: "chat", + }); + expect(parsed.options.reasoning).toBe(expectedEffort); + expect((parsed._rawBody as { reasoning: { effort?: string; summary: string } }).reasoning) + .toEqual(expectedEffort ? { effort: expectedEffort, summary: "auto" } : { summary: "auto" }); + expect(parsed.options.hideThinkingSummary).toBeUndefined(); + } + }); }); diff --git a/tests/server/server-search.test.ts b/tests/server/server-search.test.ts index 667979427ca..191ac1865ea 100644 --- a/tests/server/server-search.test.ts +++ b/tests/server/server-search.test.ts @@ -8,6 +8,8 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import { existsSync, mkdirSync} from "node:fs"; import { join } from "node:path"; +import { encodeMessage, encodeString } from "../../src/adapters/devin/cloud-direct/wire"; +import { clearComboSelectionState } from "../../src/combos"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { clearAccountNeedsReauth, clearAccountQuota } from "../../src/codex/auth-api"; import { setCodexAccountPaused } from "../../src/codex/account-pause"; @@ -18,6 +20,8 @@ import { recordCodexUpstreamOutcome, } from "../../src/codex/routing"; import { loadConfig, saveConfig } from "../../src/config"; +import { saveCredential } from "../../src/oauth/store"; +import { routeModel } from "../../src/router"; import { startServer } from "../../src/server"; import { clearRequestLogsForTests, getRequestLogEntries } from "../../src/server/request-log"; import { handleSearch, SEARCH_RESPONSE_MAX_BYTES } from "../../src/server/search"; @@ -487,6 +491,101 @@ test("returns an honest 400 when no ChatGPT forward provider is configured", asy } }); +test("routes every Devin model family through the native search RPC without a search model", async () => { + const apiKey = "devin-native-search-key"; + await saveCredential("devin", { + access: apiKey, + refresh: apiKey, + expires: Number.MAX_SAFE_INTEGER, + apiBaseUrl: "https://server.codeium.com", + }); + const requests: Array<{ url: string; body: Buffer }> = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + requests.push({ url, body: Buffer.from(init?.body as Uint8Array) }); + const item = Buffer.concat([ + encodeString(3, "https://example.test"), + encodeString(4, "Result"), + encodeString(7, "Excerpt"), + ]); + return new Response(encodeMessage(1, item), { + status: 200, + headers: { "content-type": "application/proto" }, + }); + }) as typeof fetch; + const config = { + port: 0, + defaultProvider: "devin", + providers: { + devin: { adapter: "devin", authMode: "oauth", baseUrl: "https://server.codeium.com" }, + }, + webSearchSidecar: { backend: "exa", exaApiKey: "must-not-run" }, + } as OcxConfig; + + for (const model of ["devin/claude-sonnet-5", "devin/grok-4-7", "devin/swe-2"]) { + const response = await handleSearch( + alphaSearchRequest({ model, commands: { search_query: [{ q: "current docs" }] } }), + config, + { model: "", provider: "" }, + ); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + encrypted_output: null, + results: [{ title: "Result", url: "https://example.test" }], + }); + } + expect(requests).toHaveLength(3); + expect(requests.every(request => request.url.endsWith("/exa.api_server_pb.ApiServerService/GetWebSearchResults"))).toBe(true); + expect(requests.every(request => !request.body.includes(Buffer.from("claude-sonnet-5")) + && !request.body.includes(Buffer.from("grok-4-7")) + && !request.body.includes(Buffer.from("swe-2")))).toBe(true); +}); + +test("native search route preview does not claim a round-robin combo turn", async () => { + await saveCredential("devin", { + access: "devin-native-search-key", + refresh: "devin-native-search-key", + expires: Number.MAX_SAFE_INTEGER, + apiBaseUrl: "https://server.codeium.com", + }); + globalThis.fetch = (async () => new Response(encodeMessage(1, Buffer.concat([ + encodeString(3, "https://example.test"), + encodeString(4, "Result"), + ])), { status: 200 })) as typeof fetch; + const config = { + port: 0, + defaultProvider: "devin", + providers: { + devin: { adapter: "devin", authMode: "oauth", baseUrl: "https://server.codeium.com" }, + other: { adapter: "openai-chat", baseUrl: "https://example.test/v1", apiKey: "test" }, + }, + combos: { + search: { + strategy: "round-robin", + stickyLimit: 1, + targets: [ + { provider: "devin", model: "swe-2" }, + { provider: "other", model: "other-model" }, + ], + }, + }, + } as OcxConfig; + clearComboSelectionState(); + try { + const first = routeModel(config, "combo/search"); + expect(first.providerName).toBe("devin"); + const response = await handleSearch( + alphaSearchRequest({ model: "combo/search", query: "current docs" }), + config, + { model: "", provider: "" }, + ); + expect(response.status).toBe(200); + expect(routeModel(config, "combo/search").combo?.targetIndex).toBe(first.combo?.targetIndex); + } finally { + clearComboSelectionState(); + } +}); + test("falls back to a configured exa sidecar when no ChatGPT forward provider exists", async () => { const captured: CapturedExaRequest[] = []; fakeExaUpstream(captured); diff --git a/tests/update/update-stop-first.test.ts b/tests/update/update-stop-first.test.ts index f2e1ebc809d..2f375370ae5 100644 --- a/tests/update/update-stop-first.test.ts +++ b/tests/update/update-stop-first.test.ts @@ -831,6 +831,21 @@ esac expect(recovery).toContain("currentPackageRuntimeLiveness()"); }); + test("failed-update service recovery releases the update lease before the service starts (#5760)", () => { + // The service manager starts the proxy outside this process tree, so it cannot join the + // delegated lease; it has to take the lease itself while the repair waits for it. + const start = launcherSource.indexOf("function recoverStoppedRuntimeAfterFailure("); + const recovery = launcherSource.slice(start, launcherSource.indexOf("const hasPendingTeardown", start)); + const serviceAt = recovery.indexOf('recovery.action === "service"'); + const releaseAt = recovery.indexOf("releaseUpdateLease()", serviceAt); + const replanAt = recovery.indexOf("planRecovery()", releaseAt); + const refreshAt = recovery.indexOf("refreshBackgroundServiceOrStartDirect()", replanAt); + expect(serviceAt).toBeGreaterThan(-1); + expect(releaseAt).toBeGreaterThan(serviceAt); + expect(replanAt).toBeGreaterThan(releaseAt); + expect(refreshAt).toBeGreaterThan(replanAt); + }); + test("GUI worker update children use pipe stdio so background updates do not open consoles", () => { expect(updateSource).toContain("function updateChildStdio()"); expect(updateSource).toContain('process.env.OCX_SERVICE === "1"'); diff --git a/tests/update/update-transactional.test.ts b/tests/update/update-transactional.test.ts index 9f8c95e27c0..527e6d80c4f 100644 --- a/tests/update/update-transactional.test.ts +++ b/tests/update/update-transactional.test.ts @@ -100,6 +100,22 @@ describe("#1942 transactional update", () => { expect(installArgs).not.toContain("--ignore-scripts"); }); + test("npm's strict script policy finds the stage's global root in place (#5760)", () => { + // With strict-allow-scripts, npm 11.19 plans the global tree before it creates the prefix + // layout and fails with ENOENT on /lib when the stage is bare. + const stage = stagingNpm("2.0.0"); + const result = transactionalNpmUpdate({ + packageDir, pkgName: PKG, targetVersion: "2.0.0", tag: "latest", + runNpm: (args: string[]) => { + const stageRoot = args[args.indexOf("--prefix") + 1]!; + if (process.platform !== "win32" && !existsSync(join(stageRoot, "lib"))) return { status: 254 }; + return stage(args); + }, + }); + expect(result.ok).toBe(true); + expect(liveVersion(packageDir)).toBe("2.0.0"); + }); + test("stage install failure leaves live untouched (D4 row 1)", () => { const result = transactionalNpmUpdate({ packageDir, pkgName: PKG, targetVersion: "2.0.0", tag: "latest", diff --git a/tests/web-search/devin-web-search.test.ts b/tests/web-search/devin-web-search.test.ts new file mode 100644 index 00000000000..c13d5d09ee2 --- /dev/null +++ b/tests/web-search/devin-web-search.test.ts @@ -0,0 +1,136 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { encodeMessage, encodeString } from "../../src/adapters/devin/cloud-direct/wire"; +import { mapDevinWebSearchResponse, resolveDevinWebSearchSnapshot, runDevinWebSearch } from "../../src/web-search/devin-executor"; +import { MAX_SIDECAR_RESPONSE_BYTES } from "../../src/web-search/parse"; +import { saveCredential } from "../../src/oauth/store"; +import type { OAuthAccessSnapshot } from "../../src/oauth"; + +const originalFetch = globalThis.fetch; +const snapshot: OAuthAccessSnapshot = { + provider: "devin", + accountId: "account", + generation: "generation", + accessToken: "devin-session-token$canary9876543210", + apiBaseUrl: "https://server.codeium.com", +}; + +afterEach(() => { globalThis.fetch = originalFetch; }); + +function result(fields: { url: string; title: string; summary?: string; chunk?: string }): Buffer { + const chunks = fields.chunk + ? [encodeMessage(6, encodeMessage(3, encodeString(2, fields.chunk)))] + : []; + return Buffer.concat([ + encodeString(1, "document-id"), + encodeString(3, fields.url), + encodeString(4, fields.title), + ...chunks, + ...(fields.summary ? [encodeString(7, fields.summary)] : []), + ]); +} + +describe("Devin native web search response", () => { + test("maps summary, excerpts, and safe citations without model inference", () => { + const response = Buffer.concat([ + encodeMessage(1, result({ + url: "https://docs.example/search", + title: "Search docs", + summary: "Authoritative result excerpt.", + })), + encodeMessage(1, result({ + url: "https://blog.example/update", + title: "Update", + chunk: "Fallback markdown chunk.", + })), + encodeMessage(1, result({ + url: "javascript:alert(1)", + title: "Unsafe", + summary: "must be dropped", + })), + encodeString(3, "Provider summary."), + ]); + + expect(mapDevinWebSearchResponse(response)).toEqual({ + text: "Provider summary.\n\nSearch results:\n" + + "- Search docs: Authoritative result excerpt. [https://docs.example/search]\n" + + "- Update: Fallback markdown chunk. [https://blog.example/update]", + sources: [ + { url: "https://docs.example/search", title: "Search docs" }, + { url: "https://blog.example/update", title: "Update" }, + ], + }); + }); + + test("rejects redirects, oversized bodies, malformed protobuf, and secret-bearing failures", async () => { + const cases: Array<{ response?: Response; rejection?: Error; error: string }> = [ + { response: new Response(null, { status: 302 }), error: "HTTP 302" }, + { response: new Response(new Uint8Array(MAX_SIDECAR_RESPONSE_BYTES + 1)), error: "exceeded byte bound" }, + { response: new Response(Uint8Array.from([0x0a, 0x80])), error: "malformed protobuf" }, + { rejection: new Error(`failed with ${snapshot.accessToken}`), error: "connect_error" }, + ]; + for (const fixture of cases) { + globalThis.fetch = (async (_input, init) => { + expect(init?.redirect).toBe("error"); + if (fixture.rejection) throw fixture.rejection; + return fixture.response!; + }) as typeof fetch; + const outcome = await runDevinWebSearch("query", snapshot, AbortSignal.timeout(5_000)); + expect(outcome.error).toContain(fixture.error); + expect(JSON.stringify(outcome)).not.toContain(snapshot.accessToken); + } + }); + + test("fails safely when no Devin account is signed in or its tenant is untrusted", async () => { + const previousHome = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-devin-search-auth-")); + process.env.OPENCODEX_HOME = home; + try { + expect(await resolveDevinWebSearchSnapshot("devin")).toEqual({ + error: "devin web search auth failed: no signed-in account", + }); + await saveCredential("devin", { + access: snapshot.accessToken, + refresh: snapshot.accessToken, + expires: Number.MAX_SAFE_INTEGER, + accountId: "account", + apiBaseUrl: "https://attacker.example", + }); + const resolved = await resolveDevinWebSearchSnapshot("devin"); + expect("snapshot" in resolved && resolved.snapshot.apiBaseUrl).toBeUndefined(); + globalThis.fetch = (async (input) => { + expect(String(input).startsWith("https://server.codeium.com/")).toBe(true); + return new Response(encodeMessage(1, result({ + url: "https://docs.example/search", + title: "Search docs", + summary: "Excerpt.", + }))); + }) as typeof fetch; + if ("snapshot" in resolved) { + expect((await runDevinWebSearch("query", resolved.snapshot, AbortSignal.timeout(5_000))).error).toBeUndefined(); + } + } finally { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + rmSync(home, { recursive: true, force: true }); + } + }); + + test("honors timeout and caller abort", async () => { + for (const fixture of [ + { reason: new DOMException("timeout", "TimeoutError"), expected: "timeout" }, + { reason: new DOMException("left", "AbortError"), expected: "connect_error" }, + ]) { + const caller = new AbortController(); + globalThis.fetch = ((_input, init) => new Promise((_resolve, reject) => { + if (init?.signal?.aborted) return reject(init.signal.reason); + init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true }); + })) as typeof fetch; + caller.abort(fixture.reason); + const outcome = await runDevinWebSearch("query", snapshot, caller.signal); + expect(outcome.error).toContain(fixture.expected); + } + }); +}); diff --git a/tests/web-search/web-search-recovery-kind.test.ts b/tests/web-search/web-search-recovery-kind.test.ts new file mode 100644 index 00000000000..980fb9f718b --- /dev/null +++ b/tests/web-search/web-search-recovery-kind.test.ts @@ -0,0 +1,92 @@ +import { afterEach, expect, test } from "bun:test"; +import type { AdapterEvent, OcxProviderConfig } from "../../src/types"; +import type { AdapterRequest, ProviderAdapter } from "../../src/adapters/base"; +import type { AttemptRecoveryKind } from "../../src/usage/log"; +import { parseRequest } from "../../src/responses/parser"; +import { runWithWebSearch } from "../../src/web-search/loop"; +import { createTestTranslatorBudget } from "../helpers/translator-budget"; + +// Moved out of web-search.test.ts, which sits at its file-size ratchet cap. + +const forwardProvider: OcxProviderConfig = { + adapter: "openai-responses", + baseUrl: "https://chatgpt.test/v1", + authMode: "forward", +}; + +const originalFetch = globalThis.fetch; +afterEach(() => { globalThis.fetch = originalFetch; }); + +async function collectSse(stream: ReadableStream): Promise<{ event?: string; data: Record }[]> { + const text = await new Response(stream).text(); + return text.split("\n\n") + .map(frame => frame.trim()) + .filter(frame => frame.length > 0 && frame !== "data: [DONE]") + .map(frame => { + const lines = frame.split("\n"); + const event = lines.find(line => line.startsWith("event: "))?.slice(7); + const dataLine = lines.find(line => line.startsWith("data: ")); + return { event, data: JSON.parse(dataLine?.slice(6) ?? "{}") as Record }; + }); +} + +// An account rotation and a key rotation are different operator-facing events, and the +// rotated fetch's recovery kind is the only place the attempt row records which happened. +// The loop used to hardcode `key-429` for both. +test("429 rotation reports the rotator's recovery kind", async () => { + globalThis.fetch = (() => Promise.resolve(new Response( + 'event: response.completed\ndata: {"type":"response.completed"}\n\n', + { headers: { "Content-Type": "text/event-stream" } }, + ))) as typeof fetch; + + const recoveryKindsFor = async ( + rotation: (next: ProviderAdapter) => { adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind }, + ): Promise<(AttemptRecoveryKind | undefined)[]> => { + const sends: (AttemptRecoveryKind | undefined)[] = []; + const buildRequest = (): AdapterRequest => + ({ url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }); + const firstAdapter: ProviderAdapter = { + name: "mock-429", + buildRequest, + fetchResponse: async () => new Response("rate limited", { status: 429, headers: { "retry-after": "30" } }), + async *parseStream() { /* unused */ }, + async parseResponse() { return [{ type: "done" }] as AdapterEvent[]; }, + }; + const rotatedAdapter: ProviderAdapter = { + name: "mock-rotated", + buildRequest, + fetchResponse: async () => new Response("{}", { status: 200 }), + async *parseStream() { + yield { type: "text_delta", text: "answer from rotated account" }; + yield { type: "done" }; + }, + async parseResponse() { throw new Error("parseResponse must be unreachable"); }, + }; + const response = await runWithWebSearch({ + incomingMeta: { headers: new Headers(), translatorBudget: createTestTranslatorBudget() }, + parsed: parseRequest({ model: "routed/model", input: "hi", stream: true, tools: [{ type: "web_search" }] }), + adapter: firstAdapter, + forwardProvider, + hostedTool: { type: "web_search" }, + selectedForwardHeaders: new Headers({ authorization: "Bearer token" }), + settings: { model: "gpt-5.6-luna", reasoning: "low", timeoutMs: 30_000 }, + maxSearches: 1, + onAttemptSend: recovery => { sends.push(recovery); }, + on429: () => rotation(rotatedAdapter), + }); + expect(response.status).toBe(200); + const frames = await collectSse(response.body!); + const completed = frames.find(f => f.event === "response.completed")?.data.response as Record; + const output = completed.output as { type: string; content?: { text?: string }[] }[]; + expect(output.find(o => o.type === "message")?.content?.[0]?.text).toBe("answer from rotated account"); + return sends; + }; + + // A rotator that crossed accounts says so, and the rotated send carries that kind. + expect(await recoveryKindsFor(next => ({ adapter: next, recoveryKind: "oauth-account-429" }))) + .toEqual([undefined, "oauth-account-429"]); + expect(await recoveryKindsFor(next => ({ adapter: next, recoveryKind: "anthropic-oauth-429" }))) + .toEqual([undefined, "anthropic-oauth-429"]); + expect(await recoveryKindsFor(next => ({ adapter: next, recoveryKind: "key-429" }))) + .toEqual([undefined, "key-429"]); +}); diff --git a/tests/web-search/web-search-timeout-contract.test.ts b/tests/web-search/web-search-timeout-contract.test.ts index 83d1c63f5fa..39e152d57ed 100644 --- a/tests/web-search/web-search-timeout-contract.test.ts +++ b/tests/web-search/web-search-timeout-contract.test.ts @@ -452,7 +452,7 @@ describe("web-search timeout runtime contracts", () => { connectTimeoutMs, on429: () => { rotations++; - return rotatedAdapter; + return { adapter: rotatedAdapter, recoveryKind: "key-429" as const }; }, })); diff --git a/tests/web-search/web-search.test.ts b/tests/web-search/web-search.test.ts index a027a980759..4e8a86920b9 100644 --- a/tests/web-search/web-search.test.ts +++ b/tests/web-search/web-search.test.ts @@ -1148,7 +1148,7 @@ describe("web-search sidecar native web_search_call emission", () => { if (!retryParsed) throw new Error("the loop must pass the iteration request to on429"); retryParsed._kiroAuthContext = { apiRegion: "ap-southeast-2", profileArn: "account-b" }; delete retryParsed._providerContinuation; - return rotatedAdapter; + return { adapter: rotatedAdapter, recoveryKind: "key-429" }; }, }); expect(response.status).toBe(200); @@ -1421,7 +1421,7 @@ describe("web-search sidecar native web_search_call emission", () => { settings: { model: "gpt-5.6-luna", reasoning: "low", timeoutMs: 30_000 }, maxSearches: 1, connectTimeoutMs: 100, - on429: () => rotatedAdapter, + on429: () => ({ adapter: rotatedAdapter, recoveryKind: "key-429" }), }); expect(response.status).toBe(504); const body = await response.json() as { error?: { message?: string } };