Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions apps/gateway/src/chat/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3834,7 +3834,7 @@ chat.openapi(completions, async (c) => {
}
}

let requestBody: ProviderRequestBody = await prepareRequestBody(
let requestBody: ProviderRequestBody | FormData = await prepareRequestBody(
usedProvider,
upstreamModelName,
messages as BaseMessage[],
Expand Down Expand Up @@ -3863,6 +3863,7 @@ chat.openapi(completions, async (c) => {

// Validate effective max_tokens value after prepareRequestBody
if (
!(requestBody instanceof FormData) &&
hasMaxTokens(requestBody) &&
requestBody.max_tokens !== undefined &&
finalModelInfo
Expand Down Expand Up @@ -3892,11 +3893,23 @@ chat.openapi(completions, async (c) => {
isImageGeneration &&
usedProvider === "xai" &&
url &&
!(requestBody instanceof FormData) &&
("image" in requestBody || "images" in requestBody)
) {
url = url.replace("/v1/images/generations", "/v1/images/edits");
}

// Switch OpenAI image generation endpoint to /edits when input images are present.
// prepareRequestBody returns a FormData (multipart/form-data) only for this edits flow.
if (
isImageGeneration &&
usedProvider === "openai" &&
url &&
requestBody instanceof FormData
) {
url = url.replace("/v1/images/generations", "/v1/images/edits");
}

const startTime = Date.now();
const failedEnvKeyIndicesByProvider = new Map<string, Set<number>>();
const failedTrackedKeyIdsByProvider = new Map<string, Set<string>>();
Expand Down Expand Up @@ -7811,7 +7824,9 @@ chat.openapi(completions, async (c) => {
requestId,
webSearchEnabled: !!webSearchTool,
});
headers["Content-Type"] = "application/json";
if (!(requestBody instanceof FormData)) {
headers["Content-Type"] = "application/json";
}

// Add effort beta header for Anthropic if effort parameter is specified
if (usedProvider === "anthropic" && effort !== undefined) {
Expand Down Expand Up @@ -7841,7 +7856,10 @@ chat.openapi(completions, async (c) => {
res = await fetch(url, {
method: "POST",
headers,
body: JSON.stringify(requestBody),
body:
requestBody instanceof FormData
? requestBody
: JSON.stringify(requestBody),
signal: fetchSignal,
});
} catch (error) {
Expand Down
33 changes: 33 additions & 0 deletions apps/gateway/src/chat/tools/parse-provider-response.ts
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,39 @@ export function parseProviderResponse(
break;
}
default: // OpenAI format
// Check if this is an OpenAI image generation response (e.g. gpt-image-2)
// Format: { created: number, data: [{ b64_json?: string, url?: string, revised_prompt?: string }], usage?: {...} }
if (
usedProvider === "openai" &&
json.data &&
Array.isArray(json.data) &&
json.data.length > 0 &&
(json.data[0]?.b64_json || json.data[0]?.url)
) {
const imageData = json.data;
images = imageData.map((item: any): ImageObject => {
let url: string;
if (item.b64_json) {
url = `data:image/png;base64,${item.b64_json}`;
} else {
url = item.url;
}
return {
type: "image_url",
image_url: { url },
};
});
Comment on lines +538 to +550

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Per-item fallback may produce an undefined URL.

The outer guard on Line 536 only checks json.data[0], but the map iterates every item. If a later element has neither b64_json nor url, the else branch assigns url = item.url (undefined), producing a malformed ImageObject. Consider guarding per item and skipping/erroring on invalid entries.

🛡️ Suggested hardening
-				images = imageData.map((item: any): ImageObject => {
-					let url: string;
-					if (item.b64_json) {
-						url = `data:image/png;base64,${item.b64_json}`;
-					} else {
-						url = item.url;
-					}
-					return {
-						type: "image_url",
-						image_url: { url },
-					};
-				});
+				images = imageData
+					.filter((item: any) => item?.b64_json || item?.url)
+					.map((item: any): ImageObject => ({
+						type: "image_url",
+						image_url: {
+							url: item.b64_json
+								? `data:image/png;base64,${item.b64_json}`
+								: item.url,
+						},
+					}));
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
const imageData = json.data;
images = imageData.map((item: any): ImageObject => {
let url: string;
if (item.b64_json) {
url = `data:image/png;base64,${item.b64_json}`;
} else {
url = item.url;
}
return {
type: "image_url",
image_url: { url },
};
});
const imageData = json.data;
images = imageData
.filter((item: any) => item?.b64_json || item?.url)
.map((item: any): ImageObject => ({
type: "image_url",
image_url: {
url: item.b64_json
? `data:image/png;base64,${item.b64_json}`
: item.url,
},
}));
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@apps/gateway/src/chat/tools/parse-provider-response.ts` around lines 538 -
550, The mapping over imageData can produce undefined urls because only
json.data[0] was checked; update the images construction so each item is
validated: for each item in imageData check item.b64_json and item.url, build
url = `data:image/png;base64,...` when b64_json exists, otherwise use item.url
only if defined, and skip (or log and continue) any item lacking both values
instead of returning an ImageObject with an undefined url; adjust the images
assignment that produces ImageObject entries to filter out invalid items and
ensure returned objects always have a defined image_url.url.

content = imageLabel;
finishReason = "stop";
// OpenAI gpt-image models return usage with input/output tokens
promptTokens = json.usage?.input_tokens ?? 0;
completionTokens = json.usage?.output_tokens ?? 0;
cachedTokens = json.usage?.input_tokens_details?.cached_tokens ?? null;
totalTokens =
json.usage?.total_tokens ??
(promptTokens ?? 0) + (completionTokens ?? 0);
break;
}
// Check if this is an xAI Grok Imagine image generation response
// Format: { data: [{ url: "..." }] }
if (usedProvider === "xai" && json.data && Array.isArray(json.data)) {
Expand Down
5 changes: 3 additions & 2 deletions apps/gateway/src/chat/tools/resolve-provider-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ export interface ProviderContext {
configIndex: number;
envVarName: string | undefined;
url: string;
requestBody: ProviderRequestBody;
requestBody: ProviderRequestBody | FormData;
useResponsesApi: boolean;
requestCanBeCanceled: boolean;
isImageGeneration: boolean;
Expand Down Expand Up @@ -365,7 +365,7 @@ export async function resolveProviderContext(
providers.find((p) => p.id === usedProvider)?.cancellation === true;

// --- Request body preparation ---
const requestBody: ProviderRequestBody = await prepareRequestBody(
const requestBody: ProviderRequestBody | FormData = await prepareRequestBody(
usedProvider as Provider,
upstreamModelName,
options.messages as BaseMessage[],
Expand Down Expand Up @@ -394,6 +394,7 @@ export async function resolveProviderContext(

// Post-validation of max_tokens in request body
if (
!(requestBody instanceof FormData) &&
hasMaxTokens(requestBody) &&
requestBody.max_tokens !== undefined &&
providerMappingForSelected
Expand Down
48 changes: 48 additions & 0 deletions apps/gateway/src/chat/tools/transform-response-to-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,54 @@ export function transformResponseToOpenai(
case "mistral":
case "novita":
case "openai": {
// Handle OpenAI image generation responses (e.g. gpt-image-2)
// Format: { created: number, data: [{ b64_json?: string, url?: string }], usage?: {...} }
if (
usedProvider === "openai" &&
json.data &&
Array.isArray(json.data) &&
!json.choices &&
!json.output
) {
transformedResponse = {
id: `chatcmpl-${Date.now()}`,
object: "chat.completion",
created: json.created ?? Math.floor(Date.now() / 1000),
model: `${usedProvider}/${baseModelName}`,
choices: [
{
index: 0,
message: {
role: "assistant",
content: content,
...(images && images.length > 0 && { images }),
},
finish_reason: finishReason ?? "stop",
},
],
usage: buildUsageObject(
promptTokens,
completionTokens,
totalTokens,
reasoningTokens,
cachedTokens,
costs,
showUpgradeMessage,
cacheCreationTokens,
),
metadata: buildMetadata(
requestedModel,
requestedProvider,
baseModelName,
usedProvider,
usedModel,
requestId,
routing,
usedRegion,
),
};
break;
}
// Handle OpenAI responses format transformation to chat completions format
if (json.output && Array.isArray(json.output)) {
// This is from the responses endpoint - transform to chat completions format
Expand Down
26 changes: 17 additions & 9 deletions apps/playground/src/components/playground/image-page-client.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -152,22 +152,30 @@ export default function ImagePageClient({

// Keep URL in sync with selected model(s)
useEffect(() => {
const params = new URLSearchParams(Array.from(searchParams.entries()));
// Read current URL params directly to avoid stale searchParams closure
// and to prevent an infinite loop where router.replace produces a new
// searchParams reference that re-triggers this effect (each such cycle
// causes Next.js to refetch the RSC, re-hitting /orgs forever).
const currentParams = new URLSearchParams(window.location.search);
if (comparisonMode) {
params.set("model", selectedModels.join(","));
params.set("compare", "1");
currentParams.set("model", selectedModels.join(","));
currentParams.set("compare", "1");
} else {
const primary = selectedModels[0];
if (primary) {
params.set("model", primary);
currentParams.set("model", primary);
} else {
params.delete("model");
currentParams.delete("model");
}
params.delete("compare");
currentParams.delete("compare");
}
const qs = params.toString();
router.replace(qs ? `?${qs}` : "");
}, [comparisonMode, router, searchParams, selectedModels]);
const qs = currentParams.toString();
const nextUrl = `${pathname}${qs ? `?${qs}` : ""}`;
const currentUrl = `${window.location.pathname}${window.location.search}`;
if (nextUrl !== currentUrl) {
router.replace(nextUrl, { scroll: false });
}
}, [comparisonMode, pathname, router, selectedModels]);

// Reset imageSize when model changes, clear input images when switching away from edit model
useEffect(() => {
Expand Down
26 changes: 17 additions & 9 deletions apps/playground/src/components/playground/video-page-client.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -323,22 +323,30 @@ export default function VideoPageClient({

// Keep URL in sync with selected model(s)
useEffect(() => {
const params = new URLSearchParams(Array.from(searchParams.entries()));
// Read current URL params directly to avoid stale searchParams closure
// and to prevent an infinite loop where router.replace produces a new
// searchParams reference that re-triggers this effect (each such cycle
// causes Next.js to refetch the RSC, re-hitting /orgs forever).
const currentParams = new URLSearchParams(window.location.search);
if (comparisonMode) {
params.set("model", selectedModels.join(","));
params.set("compare", "1");
currentParams.set("model", selectedModels.join(","));
currentParams.set("compare", "1");
} else {
const primary = selectedModels[0];
if (primary) {
params.set("model", primary);
currentParams.set("model", primary);
} else {
params.delete("model");
currentParams.delete("model");
}
params.delete("compare");
currentParams.delete("compare");
}
const qs = params.toString();
router.replace(qs ? `?${qs}` : "");
}, [comparisonMode, router, searchParams, selectedModels]);
const qs = currentParams.toString();
const nextUrl = `${pathname}${qs ? `?${qs}` : ""}`;
const currentUrl = `${window.location.pathname}${window.location.search}`;
if (nextUrl !== currentUrl) {
router.replace(nextUrl, { scroll: false });
}
}, [comparisonMode, pathname, router, selectedModels]);

const getModelName = useCallback(
(modelId: string) => {
Expand Down
3 changes: 3 additions & 0 deletions packages/actions/src/get-provider-endpoint.ts
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,9 @@ export function getProviderEndpoint(
}
}
case "openai": {
if (imageGenerations) {
return `${url}/v1/images/generations`;
}
// Use responses endpoint for models that support responses API
if (model) {
// Look up by model ID first, then fall back to provider modelName
Expand Down
Loading
Loading