Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions open-sse/handlers/chatCore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { buildPostCallGuardrailContext } from "./chatCore/postCallGuardrailConte
import { storeSemanticCacheResponse } from "./chatCore/semanticCacheStore.ts";
import { buildNonStreamingResponseHeaders } from "./chatCore/nonStreamingResponseHeaders.ts";
import { buildNonStreamingJsonResponse } from "./chatCore/nonStreamingJsonResponse.ts";
import { enforceOutputTokenBudget } from "./chatCore/outputTokenBudget.ts";
import { maybeConvertJsonBodyToSse } from "./chatCore/jsonBodyToSse.ts";
import { assembleStreamingResponseHeaders } from "./chatCore/streamingResponseHeaders.ts";
import { storeStreamingSemanticCacheResponse } from "./chatCore/streamingSemanticCacheStore.ts";
Expand Down Expand Up @@ -1613,6 +1614,45 @@ export async function handleChatCore({
);
}

// Re-check the concrete target after all compression passes. Combo compatibility
// filtering is advisory and may preserve an all-incompatible pool; this is the
// hard boundary that prevents a too-large prompt (or a negative token budget)
// from reaching an OpenAI-compatible upstream such as NVIDIA NIM.
const finalCompressionBody = adaptBodyForCompression(body as Record<string, unknown>).body;
Comment thread
JxnLexn marked this conversation as resolved.
Resolved
const finalMessages =
finalCompressionBody?.messages || body?.contents || body?.request?.contents || [];
Comment thread
JxnLexn marked this conversation as resolved.
Resolved
const finalEstimatedInputTokens =
estimateTokens(finalMessages) + (Array.isArray(body?.tools) ? estimateTokens(body.tools) : 0);
Comment thread
JxnLexn marked this conversation as resolved.
Resolved
const finalContextLimit = getTokenLimit(provider, effectiveModel);
const outputBudget = enforceOutputTokenBudget(
body as Record<string, unknown>,
finalEstimatedInputTokens,
finalContextLimit
Comment thread
JxnLexn marked this conversation as resolved.
Resolved
);
if (!outputBudget.ok) {
const message =
`Input exceeds the context window for ${provider}/${effectiveModel}: ` +
`estimated ${outputBudget.estimatedInputTokens} input tokens, limit ${outputBudget.contextLimit}. ` +
"Reduce the prompt or route to a model with a larger context window.";
log?.warn?.("CONTEXT", message);
trackPendingRequest(model, provider, connectionId, false);
return createErrorResult(
HTTP_STATUS.BAD_REQUEST,
message,
null,
"context_length_exceeded",
"invalid_request_error"
);
}
if (outputBudget.adjustedFields.length > 0) {
log?.info?.(
"CONTEXT",
`Adjusted invalid or oversized output token fields (${outputBudget.adjustedFields.join(", ")}); ` +
`${outputBudget.availableOutputTokens} tokens remain for output`
);
}
body = outputBudget.body;

let translatedBody = body;
const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE;
const isClaudeCodeCompatible = isClaudeCodeCompatibleProvider(provider);
Expand Down
66 changes: 66 additions & 0 deletions open-sse/handlers/chatCore/outputTokenBudget.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
export const OUTPUT_TOKEN_FIELDS = ["max_tokens", "max_completion_tokens"] as const;
Comment thread
JxnLexn marked this conversation as resolved.
Resolved

export type OutputTokenBudgetResult =
| {
ok: true;
body: Record<string, unknown>;
availableOutputTokens: number;
adjustedFields: string[];
}
| {
ok: false;
estimatedInputTokens: number;
contextLimit: number;
};

/**
* Enforce the target model's context budget immediately before translation.
*
* Compression and combo selection are best-effort: a request may still be too
* large for a concrete target, and some OpenAI-compatible gateways derive an
* internal max_tokens value by subtracting the prompt from the context window.
* Reject that target locally instead of allowing the derived value to become
* negative upstream. Positive client limits are capped to the remaining room;
* invalid numeric limits are removed.
*/
export function enforceOutputTokenBudget(
body: Record<string, unknown>,
estimatedInputTokens: number,
contextLimit: number
): OutputTokenBudgetResult {
const normalizedInputTokens = Math.max(0, Math.ceil(estimatedInputTokens));
const normalizedContextLimit = Math.max(1, Math.floor(contextLimit));
const availableOutputTokens = normalizedContextLimit - normalizedInputTokens;

if (availableOutputTokens < 1) {
return {
ok: false,
estimatedInputTokens: normalizedInputTokens,
contextLimit: normalizedContextLimit,
};
}
Comment thread
JxnLexn marked this conversation as resolved.
Resolved

let nextBody = body;
const adjustedFields: string[] = [];
for (const field of OUTPUT_TOKEN_FIELDS) {
const value = nextBody[field];
if (typeof value !== "number") continue;

const normalized = Number.isFinite(value) && value > 0 ? Math.floor(value) : null;
if (normalized === null) {
if (nextBody === body) nextBody = { ...body };
delete nextBody[field];
adjustedFields.push(field);
continue;
}

const capped = Math.min(normalized, availableOutputTokens);
if (capped !== value) {
if (nextBody === body) nextBody = { ...body };
nextBody[field] = capped;
adjustedFields.push(field);
}
}

return { ok: true, body: nextBody, availableOutputTokens, adjustedFields };
}
37 changes: 37 additions & 0 deletions tests/unit/output-token-budget.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import test from "node:test";
import assert from "node:assert/strict";

import { enforceOutputTokenBudget } from "../../open-sse/handlers/chatCore/outputTokenBudget.ts";

test("rejects a prompt that cannot leave one output token", () => {
const result = enforceOutputTokenBudget({ max_tokens: 8192 }, 527_058, 128_000);

assert.deepEqual(result, {
ok: false,
estimatedInputTokens: 527_058,
contextLimit: 128_000,
});
});

test("caps a positive output budget to the target's remaining context", () => {
const input = { messages: [], max_tokens: 12_000 };
const result = enforceOutputTokenBudget(input, 127_000, 128_000);

assert.equal(result.ok, true);
if (!result.ok) return;
assert.equal(result.body.max_tokens, 1_000);
assert.equal(input.max_tokens, 12_000, "must not mutate the shared combo request body");
});

test("removes non-positive numeric output limits before upstream dispatch", () => {
const result = enforceOutputTokenBudget(
{ max_tokens: -398_464, max_completion_tokens: 0 },
1_000,
128_000
);

assert.equal(result.ok, true);
if (!result.ok) return;
assert.equal("max_tokens" in result.body, false);
assert.equal("max_completion_tokens" in result.body, false);
});
Loading