diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index ef6e97d8993d..f7c6036885d9 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -2694,3 +2694,302 @@ it.effect("flushes managed native logs when the adapter layer shuts down", () => } }), ); + +const usageLimitRuntimeFactory = makeRuntimeFactory(); +const usageLimitLayer = it.layer( + Layer.effect( + CodexAdapter, + Effect.gen(function* () { + const codexConfig = decodeCodexSettings({}); + return yield* makeCodexAdapter(codexConfig, { + makeRuntime: usageLimitRuntimeFactory.factory, + }); + }), + ).pipe( + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), process.cwd())), + Layer.provideMerge(ServerSettingsService.layerTest()), + Layer.provideMerge(providerSessionDirectoryTestLayer), + Layer.provideMerge(NodeServices.layer), + ), +); + +const USAGE_LIMIT_NOW = "2026-01-01T00:00:00.000Z"; +const USAGE_LIMIT_NOW_SECONDS = Date.parse(USAGE_LIMIT_NOW) / 1000; +const CODEX_OUT_OF_CREDITS = + "Your workspace is out of credits. Ask your workspace owner to refill in order to continue."; + +function startUsageLimitRuntime() { + return Effect.gen(function* () { + const adapter = yield* CodexAdapter; + yield* adapter.startSession({ + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + runtimeMode: "full-access", + }); + const runtime = usageLimitRuntimeFactory.lastRuntime; + NodeAssert.ok(runtime); + return { adapter, runtime }; + }); +} + +function codexErrorNotification(input: { + readonly id: string; + readonly message: string; + readonly codexErrorInfo?: string; +}): ProviderEvent { + return { + id: asEventId(input.id), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-limit"), + createdAt: USAGE_LIMIT_NOW, + method: "error", + payload: { + threadId: "thread-1", + turnId: "turn-limit", + willRetry: false, + error: { + message: input.message, + ...(input.codexErrorInfo ? { codexErrorInfo: input.codexErrorInfo } : {}), + }, + }, + }; +} + +function codexRateLimitsNotification(input: { + readonly id: string; + readonly rateLimitReachedType?: string; + readonly primary?: { readonly usedPercent: number; readonly resetsInSeconds: number }; + readonly secondary?: { readonly usedPercent: number; readonly resetsInSeconds: number }; +}): ProviderEvent { + return { + id: asEventId(input.id), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-limit"), + createdAt: USAGE_LIMIT_NOW, + method: "account/rateLimits/updated", + payload: { + rateLimits: { + limitId: "codex", + ...(input.rateLimitReachedType ? { rateLimitReachedType: input.rateLimitReachedType } : {}), + ...(input.primary + ? { + primary: { + usedPercent: input.primary.usedPercent, + resetsAt: USAGE_LIMIT_NOW_SECONDS + input.primary.resetsInSeconds, + windowDurationMins: 300, + }, + } + : {}), + ...(input.secondary + ? { + secondary: { + usedPercent: input.secondary.usedPercent, + resetsAt: USAGE_LIMIT_NOW_SECONDS + input.secondary.resetsInSeconds, + windowDurationMins: 10_080, + }, + } + : {}), + }, + }, + }; +} + +function codexUsageLimitTurnFailed(id: string, turnId = "turn-limit"): ProviderEvent { + return { + id: asEventId(id), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId(turnId), + createdAt: USAGE_LIMIT_NOW, + method: "turn/completed", + payload: { + threadId: "thread-1", + turn: { + id: turnId, + items: [], + status: "failed", + error: { message: CODEX_OUT_OF_CREDITS, codexErrorInfo: "usageLimitExceeded" }, + }, + }, + }; +} + +usageLimitLayer("CodexAdapterLive usage limits", (it) => { + it.effect("names the exhausted window and the workspace's missing credits", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startUsageLimitRuntime(); + const eventsFiber = yield* adapter.streamEvents.pipe( + Stream.take(5), + Stream.runCollect, + Effect.forkChild, + ); + + yield* runtime.emit( + codexErrorNotification({ + id: "evt-limit-error", + message: CODEX_OUT_OF_CREDITS, + codexErrorInfo: "usageLimitExceeded", + }), + ); + yield* runtime.emit( + codexRateLimitsNotification({ + id: "evt-limit-rate-limits", + rateLimitReachedType: "workspace_owner_credits_depleted", + primary: { usedPercent: 40, resetsInSeconds: 3_600 }, + secondary: { usedPercent: 100, resetsInSeconds: 5 * 86_400 + 5 * 3_600 }, + }), + ); + yield* runtime.emit(codexUsageLimitTurnFailed("evt-limit-turn")); + // A second turn stopping on the same limit says as much as the first. + yield* runtime.emit(codexUsageLimitTurnFailed("evt-limit-turn-2", "turn-limit-2")); + + const events = Array.from(yield* Fiber.join(eventsFiber)); + const expected = + "Codex usage limit reached. The weekly limit resets in 5d 5h. The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets."; + NodeAssert.deepStrictEqual( + events.map((event) => event.type), + [ + "account.rate-limits.updated", + "runtime.error", + "turn.completed", + "runtime.error", + "turn.completed", + ], + ); + for (const event of events) { + if (event.type === "runtime.error") { + NodeAssert.equal(event.payload.message, expected); + NodeAssert.equal(event.payload.detail, CODEX_OUT_OF_CREDITS); + } + if (event.type === "turn.completed") { + NodeAssert.equal(event.payload.errorMessage, expected); + } + } + }), + ); + + it.effect("names the session window for a plan limit", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startUsageLimitRuntime(); + const eventsFiber = yield* adapter.streamEvents.pipe( + Stream.take(3), + Stream.runCollect, + Effect.forkChild, + ); + + yield* runtime.emit( + codexErrorNotification({ + id: "evt-plan-error", + message: "You've hit your usage limit.", + codexErrorInfo: "usageLimitExceeded", + }), + ); + yield* runtime.emit( + codexRateLimitsNotification({ + id: "evt-plan-rate-limits", + rateLimitReachedType: "rate_limit_reached", + primary: { usedPercent: 100, resetsInSeconds: 3 * 3_600 + 20 * 60 }, + }), + ); + yield* runtime.emit(codexUsageLimitTurnFailed("evt-plan-turn")); + + const events = Array.from(yield* Fiber.join(eventsFiber)); + const completed = events.find((event) => event.type === "turn.completed"); + NodeAssert.equal( + completed?.payload.errorMessage, + "Codex usage limit reached. The session limit resets in 3h 20m. Send the message again once the limit resets.", + ); + }), + ); + + it.effect("reads a rate-limit snapshot seen earlier in the session", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startUsageLimitRuntime(); + const eventsFiber = yield* adapter.streamEvents.pipe( + Stream.take(3), + Stream.runCollect, + Effect.forkChild, + ); + + // The window arrives long before the stop, and the update that reports the + // limit as reached carries no windows of its own. + yield* runtime.emit( + codexRateLimitsNotification({ + id: "evt-early-rate-limits", + primary: { usedPercent: 100, resetsInSeconds: 3 * 3_600 + 20 * 60 }, + }), + ); + yield* runtime.emit( + codexRateLimitsNotification({ + id: "evt-sparse-rate-limits", + rateLimitReachedType: "rate_limit_reached", + }), + ); + yield* runtime.emit(codexUsageLimitTurnFailed("evt-early-turn")); + + const events = Array.from(yield* Fiber.join(eventsFiber)); + const completed = events.find((event) => event.type === "turn.completed"); + NodeAssert.equal( + completed?.payload.errorMessage, + "Codex usage limit reached. The session limit resets in 3h 20m. Send the message again once the limit resets.", + ); + }), + ); + + it.effect("falls back to the short message without a rate-limit snapshot", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startUsageLimitRuntime(); + const eventsFiber = yield* adapter.streamEvents.pipe( + Stream.take(2), + Stream.runCollect, + Effect.forkChild, + ); + + yield* runtime.emit( + codexErrorNotification({ + id: "evt-bare-error", + message: CODEX_OUT_OF_CREDITS, + codexErrorInfo: "usageLimitExceeded", + }), + ); + yield* runtime.emit(codexUsageLimitTurnFailed("evt-bare-turn")); + + const events = Array.from(yield* Fiber.join(eventsFiber)); + const expected = "Codex usage limit reached. Send the message again once the limit resets."; + NodeAssert.deepStrictEqual( + events.map((event) => event.type), + ["runtime.error", "turn.completed"], + ); + const runtimeError = events.find((event) => event.type === "runtime.error"); + NodeAssert.equal(runtimeError?.payload.message, expected); + const completed = events.find((event) => event.type === "turn.completed"); + NodeAssert.equal(completed?.payload.errorMessage, expected); + }), + ); + + it.effect("still relays other provider errors as they arrive", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startUsageLimitRuntime(); + const firstEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe(Effect.forkChild); + + yield* runtime.emit( + codexErrorNotification({ + id: "evt-other-error", + message: "Codex is temporarily unavailable.", + codexErrorInfo: "internalServerError", + }), + ); + + const first = yield* Fiber.join(firstEventFiber); + NodeAssert.equal(first._tag, "Some"); + if (first._tag !== "Some" || first.value.type !== "runtime.error") return; + NodeAssert.equal(first.value.payload.message, "Codex is temporarily unavailable."); + NodeAssert.equal(first.value.payload.class, "provider_error"); + }), + ); +}); diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 5e2244336afc..2d88e58dc1fb 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -71,7 +71,12 @@ import { } from "./CodexSessionRuntime.ts"; import { type EventNdjsonLogger, makeEventNdjsonLogger } from "./EventNdjsonLogger.ts"; import { resolveCodexLaunchArgs } from "./codexLaunchArgs.ts"; -import { codexRateLimitsToUpdate } from "./codexUsageLimits.ts"; +import { + type CodexRateLimitSnapshot, + codexRateLimitsToUpdate, + codexUsageLimitMessage, + mergeCodexRateLimits, +} from "./codexUsageLimits.ts"; const isCodexAppServerProcessExitedError = Schema.is(CodexErrors.CodexAppServerProcessExitedError); const isCodexAppServerTransportError = Schema.is(CodexErrors.CodexAppServerTransportError); const isCodexSessionRuntimeThreadIdMissingError = Schema.is( @@ -2282,6 +2287,12 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( : {}), }; const turnTokenUsage = makeCodexTurnTokenUsageState(); + // Codex reports a usage-limit stop as OpenAI's own sentence, which on a + // Business workspace blames credits for a window that ran out. The + // snapshot naming that window arrives in its own notification, before or + // after the stop and often sparse, so keep the session's merged view of + // it and read it when a turn fails on the limit. + let rateLimits: CodexRateLimitSnapshot | undefined; const sessionScope = yield* Scope.make("sequential"); let sessionScopeTransferred = false; yield* Effect.addFinalizer(() => @@ -2339,12 +2350,56 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( } } - const runtimeEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => { + if (event.method === "account/rateLimits/updated") { + const limitsPayload = readPayload( + EffectCodexSchema.V2AccountRateLimitsUpdatedNotification, + event.payload, + ); + if (limitsPayload) { + rateLimits = mergeCodexRateLimits(rateLimits, limitsPayload.rateLimits); + } + } else if (event.method === "error") { + const errorPayload = readPayload( + EffectCodexSchema.V2ErrorNotification, + event.payload, + ); + // The failed `turn/completed` repeats this sentence and is answered + // below; relaying both would show the limit twice. + if (errorPayload?.error.codexErrorInfo === "usageLimitExceeded") return; + } + + let usageLimitError: ProviderRuntimeEvent | undefined; + let usageLimitMessage: string | undefined; + if (event.method === "turn/completed") { + const completedPayload = readPayload( + EffectCodexSchema.V2TurnCompletedNotification, + event.payload, + ); + const turnError = + completedPayload?.turn.status === "failed" + ? completedPayload.turn.error + : undefined; + if (turnError?.codexErrorInfo === "usageLimitExceeded") { + usageLimitMessage = codexUsageLimitMessage(rateLimits, event.createdAt); + usageLimitError = { + ...runtimeEventBase(event, event.threadId), + type: "runtime.error", + payload: { + message: usageLimitMessage, + class: "provider_error", + ...(turnError.message ? { detail: turnError.message } : {}), + }, + }; + } + } + + const mappedEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => { if (runtimeEvent.type === "turn.completed" && runtimeEvent.turnId) { return { ...runtimeEvent, payload: { ...runtimeEvent.payload, + ...(usageLimitMessage ? { errorMessage: usageLimitMessage } : {}), tokenUsage: completeCodexTurnTokenUsage( turnTokenUsage, String(runtimeEvent.turnId), @@ -2368,6 +2423,9 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( } return runtimeEvent; }); + const runtimeEvents = usageLimitError + ? [usageLimitError, ...mappedEvents] + : mappedEvents; if (runtimeEvents.length === 0) { yield* Effect.logDebug("ignoring unhandled Codex provider event", { method: event.method, diff --git a/apps/server/src/provider/Layers/codexUsageLimits.test.ts b/apps/server/src/provider/Layers/codexUsageLimits.test.ts index 9de27ef75c25..37006789986c 100644 --- a/apps/server/src/provider/Layers/codexUsageLimits.test.ts +++ b/apps/server/src/provider/Layers/codexUsageLimits.test.ts @@ -6,6 +6,8 @@ import { codexRateLimitsToLimits, codexRateLimitsToUpdate, codexResetCreditsToContract, + codexUsageLimitMessage, + mergeCodexRateLimits, } from "./codexUsageLimits.ts"; const checkedAt = "2026-07-18T10:00:00.000Z"; @@ -201,3 +203,97 @@ describe("codexResetCreditsToContract", () => { ).toEqual({ availableCount: 1 }); }); }); + +describe("codexUsageLimitMessage", () => { + const at = "2026-01-01T00:00:00.000Z"; + const atSeconds = Date.parse(at) / 1000; + + it("names the exhausted window and the workspace's missing credits", () => { + expect( + codexUsageLimitMessage( + { + limitId: "codex", + rateLimitReachedType: "workspace_owner_credits_depleted", + primary: { usedPercent: 40, resetsAt: atSeconds + 3_600, windowDurationMins: 300 }, + secondary: { + usedPercent: 100, + resetsAt: atSeconds + 5 * 86_400 + 5 * 3_600, + windowDurationMins: 10_080, + }, + }, + at, + ), + ).toBe( + "Codex usage limit reached. The weekly limit resets in 5d 5h. The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets.", + ); + }); + + it("points a reached spend cap at the workspace owner", () => { + expect( + codexUsageLimitMessage( + { + limitId: "codex", + rateLimitReachedType: "workspace_member_usage_limit_reached", + primary: { + usedPercent: 100, + resetsAt: atSeconds + 3 * 3_600 + 20 * 60, + windowDurationMins: 300, + }, + }, + at, + ), + ).toBe( + "Codex usage limit reached. The session limit resets in 3h 20m. The workspace spend limit is reached: ask your workspace owner to raise it, or send the message again once the limit resets.", + ); + }); + + it("names no window when credits run out without one", () => { + expect( + codexUsageLimitMessage( + { limitId: "codex", rateLimitReachedType: "workspace_member_credits_depleted" }, + at, + ), + ).toBe( + "Codex usage limit reached. The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets.", + ); + }); + + it("says only what it knows without a snapshot", () => { + expect(codexUsageLimitMessage(undefined, at)).toBe( + "Codex usage limit reached. Send the message again once the limit resets.", + ); + }); +}); + +describe("mergeCodexRateLimits", () => { + it("keeps windows an update does not carry", () => { + const merged = mergeCodexRateLimits( + { + limitId: "codex", + planType: "business", + primary: { usedPercent: 100, resetsAt: 1_800_000_000, windowDurationMins: 300 }, + }, + { rateLimitReachedType: "rate_limit_reached" }, + ); + + expect(merged).toEqual({ + limitId: "codex", + planType: "business", + rateLimitReachedType: "rate_limit_reached", + primary: { usedPercent: 100, resetsAt: 1_800_000_000, windowDurationMins: 300 }, + }); + }); + + it("ignores a model-specific snapshot so it cannot replace the main allowance", () => { + const main = { + limitId: "codex", + primary: { usedPercent: 100, resetsAt: 1_800_000_000, windowDurationMins: 300 }, + }; + expect( + mergeCodexRateLimits(main, { + limitId: "spark", + primary: { usedPercent: 3, resetsAt: 1_800_000_000, windowDurationMins: 300 }, + }), + ).toBe(main); + }); +}); diff --git a/apps/server/src/provider/Layers/codexUsageLimits.ts b/apps/server/src/provider/Layers/codexUsageLimits.ts index 32862cdb337d..66b9511b1e44 100644 --- a/apps/server/src/provider/Layers/codexUsageLimits.ts +++ b/apps/server/src/provider/Layers/codexUsageLimits.ts @@ -28,6 +28,7 @@ interface CodexRateLimitWindow { export interface CodexRateLimitSnapshot { readonly limitId?: string | null; readonly planType?: string | null; + readonly rateLimitReachedType?: string | null; readonly primary?: CodexRateLimitWindow | null; readonly secondary?: CodexRateLimitWindow | null; } @@ -157,3 +158,77 @@ export function codexRateLimitsFailureMessage(error: CodexErrors.CodexAppServerE return "Codex did not answer the usage request."; } } + +/** + * Codex sends `account/rateLimits/updated` as a partial view of the snapshot: a + * field the update omits keeps the value observed earlier in the session, so a + * later notification that only names the limit it reached must not drop the + * windows an earlier one carried. + */ +export function mergeCodexRateLimits( + previous: CodexRateLimitSnapshot | undefined, + update: CodexRateLimitSnapshot, +): CodexRateLimitSnapshot | undefined { + // Model-specific snapshots (such as Spark) describe a different allowance + // and must not overwrite the main one, the same rule the usage rows apply. + if (update.limitId && update.limitId !== "codex") return previous; + if (!previous) return update; + return { + ...previous, + ...(update.limitId !== undefined ? { limitId: update.limitId } : {}), + ...(update.planType !== undefined ? { planType: update.planType } : {}), + ...(update.rateLimitReachedType !== undefined + ? { rateLimitReachedType: update.rateLimitReachedType } + : {}), + ...(update.primary !== undefined ? { primary: update.primary } : {}), + ...(update.secondary !== undefined ? { secondary: update.secondary } : {}), + }; +} + +/** Coarse remaining wait, matching how the usage rows read: `5d 5h`, `3h 20m`, `12m`. */ +function formatCodexUsageLimitWait(waitMs: number): string { + const totalMinutes = Math.ceil(waitMs / 60_000); + const days = Math.floor(totalMinutes / (24 * 60)); + const hours = Math.floor((totalMinutes % (24 * 60)) / 60); + const minutes = totalMinutes % 60; + if (days > 0) return hours === 0 ? `${days}d` : `${days}d ${hours}h`; + if (hours === 0) return `${totalMinutes}m`; + return minutes === 0 ? `${hours}h` : `${hours}h ${minutes}m`; +} + +function codexUsageLimitNextStep(rateLimitReachedType: string | null | undefined): string { + switch (rateLimitReachedType) { + case "workspace_owner_credits_depleted": + case "workspace_member_credits_depleted": + return " The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets."; + case "workspace_owner_usage_limit_reached": + case "workspace_member_usage_limit_reached": + return " The workspace spend limit is reached: ask your workspace owner to raise it, or send the message again once the limit resets."; + default: + return " Send the message again once the limit resets."; + } +} + +/** + * The message a usage-limit stop shows instead of the provider sentence, which + * on a Business workspace blames credits for a window that simply ran out. The + * window named is the exhausted one that has yet to reset, latest first; `atIso` + * is the stopping event's timestamp, not the wall clock. + */ +export function codexUsageLimitMessage( + snapshot: CodexRateLimitSnapshot | undefined, + atIso: string, +): string { + const atMs = Date.parse(atIso); + const windows = snapshot && Number.isFinite(atMs) ? codexRateLimitsToWindows(snapshot) : []; + let reset = ""; + let latestResetMs = Number.NEGATIVE_INFINITY; + for (const window of windows) { + if (window.usedPercent < 100 || !window.resetsAt) continue; + const resetMs = Date.parse(window.resetsAt); + if (!Number.isFinite(resetMs) || resetMs <= atMs || resetMs <= latestResetMs) continue; + latestResetMs = resetMs; + reset = ` The ${window.kind} limit resets in ${formatCodexUsageLimitWait(resetMs - atMs)}.`; + } + return `Codex usage limit reached.${reset}${codexUsageLimitNextStep(snapshot?.rateLimitReachedType)}`; +} diff --git a/docs/user/providers-codex.md b/docs/user/providers-codex.md index 085a8e55a257..06c59c6f6aee 100644 --- a/docs/user/providers-codex.md +++ b/docs/user/providers-codex.md @@ -65,6 +65,13 @@ in the thread on web, desktop, or mobile. Some tools offer access for one reques the current session, or permanently. See [Permission modes](./permission-modes.md) for command and file approvals. +## Codex says I hit a usage limit + +When Codex stops on a usage limit, the thread names the window that ran out and +when it resets, when Codex reports them. Send the message again after the reset. On a workspace plan the +message also says whether your workspace owner needs to add credits or raise the +spend limit to continue sooner. + ## Send feedback to OpenAI In an existing Codex thread, send `/feedback` with an optional description, for