Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixes/0000-terminal-local-process-pressure.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **fix(resilience):** Stop combo retries and fallback on process-wide `resource_pressure` / `heap_pressure` 503s, preserving the original retry hint and healthy account affinity. Cancel in-flight hedges and release their delay timers. (#0000 — thanks @aldoeliacim)
10 changes: 8 additions & 2 deletions open-sse/services/combo/comboAttemptLoop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -468,10 +468,15 @@ export async function dispatchWithCooldownRetry(opts: {
dispatchPosition + 1 < dispatchIndices.length
) {
const hedgeDelay = resolveDelayMs(deps.config.hedgeDelayMs, 500);
let hedgeTimer: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<void>((r) => {
setTimeout(r, hedgeDelay);
hedgeTimer = setTimeout(r, hedgeDelay);
});
await Promise.race([task, globalPromise, timeoutPromise, loopSafetyPromise]);
try {
await Promise.race([task, globalPromise, timeoutPromise, loopSafetyPromise]);
} finally {
clearTimeout(hedgeTimer);
}
} else {
await Promise.race([task, globalPromise, loopSafetyPromise]);
}
Expand Down Expand Up @@ -711,6 +716,7 @@ export async function dispatchWithCooldownRetry(opts: {
}
);
if (compatFallbackResult) {
if (!compatFallbackResult.ok) return compatFallbackResult;
recordComboRequest(deps.combo.name, null, {
success: true,
latencyMs: Date.now() - deps.startTime,
Expand Down
8 changes: 8 additions & 0 deletions open-sse/services/combo/comboCompatFallback.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { readAttemptFailure } from "./attemptFailure.ts";
import { isLocalProcessPressureResponse } from "./comboPredicates.ts";
import type {
ComboLogger,
HandleSingleModel,
Expand Down Expand Up @@ -37,6 +39,7 @@ export interface CompatFallbackContext {
* upstream `Response`. Targets that are unavailable, in cooldown, or return a non-ok
* response are skipped. Returns `null` when no rejected target yields a success, so the
* caller can fall through to its existing error/503 path with an unchanged error surface.
* Process-wide pressure instead returns its original terminal response, never a success.
*/
export async function attemptCompatRejectedFallback(
rejectedTargets: ResolvedComboTarget[],
Expand Down Expand Up @@ -75,6 +78,11 @@ export async function attemptCompatRejectedFallback(
effectiveComboStrategy: ctx.strategy,
fallbackAttempts: i,
});
if (
result.status === 503 &&
isLocalProcessPressureResponse(result, (await readAttemptFailure(result)).errorBody)
)
return result;
if (result.ok) {
ctx.log.info("COMBO", `Last-resort compat fallback succeeded via ${target.modelStr}`);
return result;
Expand Down
9 changes: 9 additions & 0 deletions open-sse/services/combo/comboPredicates.ts
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,15 @@ export function isAccountSemaphoreCapacityFailure(error?: {
);
}

/** Process-wide shedding cannot recover by retrying another combo target. */
export function isLocalProcessPressureResponse(response: Response, body: unknown): boolean {
if (response.status !== 503 || !body || typeof body !== "object") return false;
const error = (body as Record<string, unknown>).error;
if (!error || typeof error !== "object") return false;
const code = (error as Record<string, unknown>).code;
return code === "resource_pressure" || code === "heap_pressure";
}

Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
export function isComboTargetTimeoutFailure(error?: {
code?: string | null;
type?: string | null;
Expand Down
13 changes: 12 additions & 1 deletion open-sse/services/combo/dispatchPrelude.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,13 @@ import { errorResponse, errorResponseWithComboDiagnostics } from "../../utils/er
import { parseModel } from "../model.ts";
import { handlePipelineChat, type PipelineStep } from "../pipeline.ts";
import type { resolveComboSetupConfig } from "../comboConfig.ts";
import { clampComboDepth, clampGlobalAttempts, resolveDelayMs } from "./comboPredicates.ts";
import {
clampComboDepth,
clampGlobalAttempts,
resolveDelayMs,
isLocalProcessPressureResponse,
} from "./comboPredicates.ts";
import { readAttemptFailure } from "./attemptFailure.ts";
import {
deriveRequestCompatibilityRequirements,
isVisionIncompatibleTarget,
Expand Down Expand Up @@ -284,6 +290,11 @@ async function evaluatePinnedResponse(args: {
);
return null;
}
if (
pinnedResult.status === 503 &&
isLocalProcessPressureResponse(pinnedResult, (await readAttemptFailure(pinnedResult)).errorBody)
)
return pinnedResult;
const pinnedStatus = pinnedResult.status || 500;
if (![401, 408, 429, 500, 502, 503, 504].includes(pinnedStatus)) {
return pinnedResult;
Expand Down
6 changes: 6 additions & 0 deletions open-sse/services/combo/executeTargetAttempt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ import {
isHedgeCancelledSentinel,
isAccountSemaphoreCapacityFailure,
isLocalQueueCapacityErrorBody,
isLocalProcessPressureResponse,
toRecordedTarget,
resolveDelayMs,
isModelScoped400,
Expand Down Expand Up @@ -779,6 +780,11 @@ export async function executeTargetAttempt(

const failure = await readAttemptFailure(result, { log: deps.log, modelStr });
const { errorText, errorBody, structuredError } = failure;
// Return the original verdict before retry hints, health or affinity mutations.
// The attempt loop treats this as fatal and cancels already-running hedges.
if (isLocalProcessPressureResponse(result, errorBody)) {
return { ok: false, response: result };
}
const retryAfter = resolveAttemptRetryAfter(failure);

// Track earliest retryAfter
Expand Down
5 changes: 5 additions & 0 deletions open-sse/services/combo/roundRobinCombo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ import {
isLocalKeyPolicyBreachErrorBody,
isAccountSemaphoreCapacityFailure,
isLocalQueueCapacityErrorBody,
isLocalProcessPressureResponse,
isProviderCircuitOpenResult,
toRecordedTarget,
getExhaustedTargetSkipReason,
Expand Down Expand Up @@ -1112,6 +1113,9 @@ export async function handleRoundRobinCombo({
} catch {
logRetryHintUnreadable(log, "COMBO-RR", modelStr, result.status, "clone failed");
}
// Process-wide shedding is terminal, not a target failure. Preserve the
// original body/headers and leave health, semaphore cooldown and pins alone.
if (isLocalProcessPressureResponse(result, errorBody)) return result;
retryAfter ||= readProseRetryAfter(bodyText); // #13672 opt-in prose hints

if (result.status === 499) {
Expand Down Expand Up @@ -1453,6 +1457,7 @@ export async function handleRoundRobinCombo({
strategy: "round-robin",
});
if (compatFallbackResult) {
if (!compatFallbackResult.ok) return compatFallbackResult;
recordComboRequest(combo.name, null, {
success: true,
latencyMs: Date.now() - startTime,
Expand Down
14 changes: 13 additions & 1 deletion open-sse/services/combo/runtimeUnits.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,12 @@ import { waitForCooldownAwareRetry } from "../../../src/sse/services/cooldownAwa
import { buildContextBudgetResponse, readBudgetFailure } from "./budgetExhaustion.ts";
import type { ComboDiagnostics } from "../../utils/error.ts";
import { recordComboRequest } from "../comboMetrics.ts";
import { resolveDelayMs, requestScopedReplayKey } from "./comboPredicates.ts";
import {
resolveDelayMs,
requestScopedReplayKey,
isLocalProcessPressureResponse,
} from "./comboPredicates.ts";
import { readAttemptFailure } from "./attemptFailure.ts";
import {
isProviderSafetyStreamFailure,
qualityValidationFailure,
Expand Down Expand Up @@ -359,6 +364,13 @@ export async function executeRuntimeUnitCombo(args: {
});
return { response, unit };
}
if (
response.status === 503 &&
isLocalProcessPressureResponse(response, (await readAttemptFailure(response)).errorBody)
) {
// Do not replay a shed child or replace its response with an aggregate.
return { response, unit };
}
lastResponse = response;
if (response.ok) {
if (unit.kind === "combo-ref") {
Expand Down
1 change: 1 addition & 0 deletions stryker.conf.json
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,7 @@
"tests/unit/combo-identical-error-streak.test.ts",
"tests/unit/combo-priority-compat-fallback.test.ts",
"tests/unit/combo-priority-quota-exhaustion-cutoff-5923.test.ts",
"tests/unit/combo-process-pressure-terminal.test.ts",
"tests/unit/combo-provider-cooldown-sibling.test.ts",
"tests/unit/combo-provider-cooldown.test.ts",
"tests/unit/combo-provider-diversity-wiring.test.ts",
Expand Down
Loading
Loading