From 338a87c7ca191d332177588f0149e6273d29873b Mon Sep 17 00:00:00 2001 From: smakosh Date: Tue, 21 Apr 2026 22:55:09 +0200 Subject: [PATCH 1/5] feat(models): add gpt-image-2 from openai Add GPT Image 2 with token-based pricing: text $5/$10 per 1M in/out, cached text $1.25/1M, image input $8/1M, image output $30/1M. Routes through the image generations pipeline. Co-Authored-By: Claude Opus 4.6 --- packages/models/src/models/openai.ts | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/models/src/models/openai.ts b/packages/models/src/models/openai.ts index 2c273517ff..813fd9c727 100644 --- a/packages/models/src/models/openai.ts +++ b/packages/models/src/models/openai.ts @@ -1807,4 +1807,33 @@ export const openaiModels = [ }, ], }, + { + id: "gpt-image-2", + name: "GPT Image 2", + description: + "OpenAI's next-generation image model with improved quality and prompt adherence, supporting text and image inputs.", + family: "openai", + output: ["image"], + releasedAt: new Date("2026-04-21"), + providers: [ + { + test: "skip", + providerId: "openai", + modelName: "gpt-image-2", + inputPrice: 5 / 1e6, + outputPrice: 10 / 1e6, + cachedInputPrice: 1.25 / 1e6, + imageInputPrice: 8 / 1e6, + imageOutputPrice: 30 / 1e6, + requestPrice: 0, + contextSize: 128000, + maxOutput: 4096, + streaming: false, + vision: true, + tools: false, + jsonOutput: false, + imageGenerations: true, + }, + ], + }, ] as const satisfies ModelDefinition[]; From a186396740e55789d1e9537df36a457577c0f5e7 Mon Sep 17 00:00:00 2001 From: smakosh Date: Wed, 22 Apr 2026 23:59:14 +0200 Subject: [PATCH 2/5] fix(gpt-image): wire openai image-gen routing end-to-end - Route openai imageGenerations to /v1/images/generations - Transform chat-shape payload to images API shape and normalize "1K"/aspect-ratio into 1024x1024/1024x1536/1536x1024 - Parse { data:[{b64_json|url}] } responses back into images - Swap to /v1/images/edits when an input image is present Also fix infinite /orgs refetch loop: image/video page URL-sync effects depended on searchParams, causing router.replace to produce a new ref that retriggered the effect and refetched the RSC forever. Match chat-page-client pattern (read window.location.search, compare before replace, scroll:false). --- apps/gateway/src/chat/chat.ts | 10 +++ .../src/chat/tools/parse-provider-response.ts | 33 ++++++++ .../playground/image-page-client.tsx | 26 +++--- .../playground/video-page-client.tsx | 26 +++--- packages/actions/src/get-provider-endpoint.ts | 3 + packages/actions/src/prepare-request-body.ts | 80 +++++++++++++++++++ 6 files changed, 160 insertions(+), 18 deletions(-) diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index a8bbebc71c..6ff397aa05 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -3897,6 +3897,16 @@ chat.openapi(completions, async (c) => { url = url.replace("/v1/images/generations", "/v1/images/edits"); } + // Switch OpenAI image generation endpoint to /edits when input images are present + if ( + isImageGeneration && + usedProvider === "openai" && + url && + "image" in requestBody + ) { + url = url.replace("/v1/images/generations", "/v1/images/edits"); + } + const startTime = Date.now(); const failedEnvKeyIndicesByProvider = new Map>(); const failedTrackedKeyIdsByProvider = new Map>(); diff --git a/apps/gateway/src/chat/tools/parse-provider-response.ts b/apps/gateway/src/chat/tools/parse-provider-response.ts index f772b035e9..f2e6b2fc44 100644 --- a/apps/gateway/src/chat/tools/parse-provider-response.ts +++ b/apps/gateway/src/chat/tools/parse-provider-response.ts @@ -526,6 +526,39 @@ export function parseProviderResponse( break; } default: // OpenAI format + // Check if this is an OpenAI image generation response (e.g. gpt-image-2) + // Format: { created: number, data: [{ b64_json?: string, url?: string, revised_prompt?: string }], usage?: {...} } + if ( + usedProvider === "openai" && + json.data && + Array.isArray(json.data) && + json.data.length > 0 && + (json.data[0]?.b64_json || json.data[0]?.url) + ) { + const imageData = json.data; + images = imageData.map((item: any): ImageObject => { + let url: string; + if (item.b64_json) { + url = `data:image/png;base64,${item.b64_json}`; + } else { + url = item.url; + } + return { + type: "image_url", + image_url: { url }, + }; + }); + content = imageLabel; + finishReason = "stop"; + // OpenAI gpt-image models return usage with input/output tokens + promptTokens = json.usage?.input_tokens ?? 0; + completionTokens = json.usage?.output_tokens ?? 0; + cachedTokens = json.usage?.input_tokens_details?.cached_tokens ?? null; + totalTokens = + json.usage?.total_tokens ?? + (promptTokens ?? 0) + (completionTokens ?? 0); + break; + } // Check if this is an xAI Grok Imagine image generation response // Format: { data: [{ url: "..." }] } if (usedProvider === "xai" && json.data && Array.isArray(json.data)) { diff --git a/apps/playground/src/components/playground/image-page-client.tsx b/apps/playground/src/components/playground/image-page-client.tsx index 59b3eb2eb1..e69503ca2c 100644 --- a/apps/playground/src/components/playground/image-page-client.tsx +++ b/apps/playground/src/components/playground/image-page-client.tsx @@ -152,22 +152,30 @@ export default function ImagePageClient({ // Keep URL in sync with selected model(s) useEffect(() => { - const params = new URLSearchParams(Array.from(searchParams.entries())); + // Read current URL params directly to avoid stale searchParams closure + // and to prevent an infinite loop where router.replace produces a new + // searchParams reference that re-triggers this effect (each such cycle + // causes Next.js to refetch the RSC, re-hitting /orgs forever). + const currentParams = new URLSearchParams(window.location.search); if (comparisonMode) { - params.set("model", selectedModels.join(",")); - params.set("compare", "1"); + currentParams.set("model", selectedModels.join(",")); + currentParams.set("compare", "1"); } else { const primary = selectedModels[0]; if (primary) { - params.set("model", primary); + currentParams.set("model", primary); } else { - params.delete("model"); + currentParams.delete("model"); } - params.delete("compare"); + currentParams.delete("compare"); } - const qs = params.toString(); - router.replace(qs ? `?${qs}` : ""); - }, [comparisonMode, router, searchParams, selectedModels]); + const qs = currentParams.toString(); + const nextUrl = `${pathname}${qs ? `?${qs}` : ""}`; + const currentUrl = `${window.location.pathname}${window.location.search}`; + if (nextUrl !== currentUrl) { + router.replace(nextUrl, { scroll: false }); + } + }, [comparisonMode, pathname, router, selectedModels]); // Reset imageSize when model changes, clear input images when switching away from edit model useEffect(() => { diff --git a/apps/playground/src/components/playground/video-page-client.tsx b/apps/playground/src/components/playground/video-page-client.tsx index 2482560e92..849be130d0 100644 --- a/apps/playground/src/components/playground/video-page-client.tsx +++ b/apps/playground/src/components/playground/video-page-client.tsx @@ -323,22 +323,30 @@ export default function VideoPageClient({ // Keep URL in sync with selected model(s) useEffect(() => { - const params = new URLSearchParams(Array.from(searchParams.entries())); + // Read current URL params directly to avoid stale searchParams closure + // and to prevent an infinite loop where router.replace produces a new + // searchParams reference that re-triggers this effect (each such cycle + // causes Next.js to refetch the RSC, re-hitting /orgs forever). + const currentParams = new URLSearchParams(window.location.search); if (comparisonMode) { - params.set("model", selectedModels.join(",")); - params.set("compare", "1"); + currentParams.set("model", selectedModels.join(",")); + currentParams.set("compare", "1"); } else { const primary = selectedModels[0]; if (primary) { - params.set("model", primary); + currentParams.set("model", primary); } else { - params.delete("model"); + currentParams.delete("model"); } - params.delete("compare"); + currentParams.delete("compare"); } - const qs = params.toString(); - router.replace(qs ? `?${qs}` : ""); - }, [comparisonMode, router, searchParams, selectedModels]); + const qs = currentParams.toString(); + const nextUrl = `${pathname}${qs ? `?${qs}` : ""}`; + const currentUrl = `${window.location.pathname}${window.location.search}`; + if (nextUrl !== currentUrl) { + router.replace(nextUrl, { scroll: false }); + } + }, [comparisonMode, pathname, router, selectedModels]); const getModelName = useCallback( (modelId: string) => { diff --git a/packages/actions/src/get-provider-endpoint.ts b/packages/actions/src/get-provider-endpoint.ts index f26d1082b6..ac51bac6da 100644 --- a/packages/actions/src/get-provider-endpoint.ts +++ b/packages/actions/src/get-provider-endpoint.ts @@ -403,6 +403,9 @@ export function getProviderEndpoint( } } case "openai": { + if (imageGenerations) { + return `${url}/v1/images/generations`; + } // Use responses endpoint for models that support responses API if (model) { // Look up by model ID first, then fall back to provider modelName diff --git a/packages/actions/src/prepare-request-body.ts b/packages/actions/src/prepare-request-body.ts index 89fee05b9c..baee79d980 100644 --- a/packages/actions/src/prepare-request-body.ts +++ b/packages/actions/src/prepare-request-body.ts @@ -579,6 +579,86 @@ export async function prepareRequestBody( reasoning_max_tokens?: number, useResponsesApi?: boolean, ): Promise { + // Handle OpenAI image generation models (e.g. gpt-image-2) + if (imageGenerations && usedProvider === "openai") { + // Extract prompt and image URLs from last user message + const lastUserMessage = [...messages] + .reverse() + .find((m) => m.role === "user"); + let prompt = ""; + const imageUrls: string[] = []; + if (lastUserMessage) { + if (typeof lastUserMessage.content === "string") { + prompt = lastUserMessage.content; + } else if (Array.isArray(lastUserMessage.content)) { + for (const part of lastUserMessage.content) { + if (part.type === "text" && part.text) { + prompt += (prompt ? "\n" : "") + part.text; + } else if (part.type === "image_url" && part.image_url) { + const url = + typeof part.image_url === "string" + ? part.image_url + : part.image_url.url; + if (url) { + imageUrls.push(url); + } + } + } + } + } + + // Normalize size to OpenAI gpt-image accepted values. + // gpt-image-1/2 accepts: "1024x1024", "1024x1536", "1536x1024", "auto". + const rawSize = image_config?.image_size; + const aspectRatio = image_config?.aspect_ratio; + let openaiSize: string | undefined; + if (rawSize) { + const normalized = rawSize.toLowerCase(); + if ( + normalized === "1024x1024" || + normalized === "1024x1536" || + normalized === "1536x1024" || + normalized === "auto" + ) { + openaiSize = rawSize; + } else if (normalized === "1k") { + // Map resolution presets with aspect ratio when available + if (aspectRatio === "16:9" || aspectRatio === "3:2") { + openaiSize = "1536x1024"; + } else if (aspectRatio === "9:16" || aspectRatio === "2:3") { + openaiSize = "1024x1536"; + } else { + openaiSize = "1024x1024"; + } + } else { + openaiSize = "auto"; + } + } else if (aspectRatio && aspectRatio !== "auto") { + if (aspectRatio === "16:9" || aspectRatio === "3:2") { + openaiSize = "1536x1024"; + } else if (aspectRatio === "9:16" || aspectRatio === "2:3") { + openaiSize = "1024x1536"; + } else if (aspectRatio === "1:1") { + openaiSize = "1024x1024"; + } + } + + const openaiImageRequest: any = { + model: usedModel, + prompt, + ...(openaiSize && { size: openaiSize }), + ...(image_config?.n && { n: image_config.n }), + }; + + if (imageUrls.length > 0) { + // For edits flow (handled by URL swap in chat.ts) + openaiImageRequest.image = + imageUrls.length === 1 ? imageUrls[0] : imageUrls; + } + + return openaiImageRequest; + } + // Handle xAI image generation models if (imageGenerations && usedProvider === "xai") { // Extract prompt and image URLs from last user message From bdc8ca8e40e4cd4647726eae13528256e4dd11f9 Mon Sep 17 00:00:00 2001 From: smakosh Date: Thu, 23 Apr 2026 22:23:43 +0200 Subject: [PATCH 3/5] fix(gpt-image): transform openai image-gen response Co-Authored-By: Claude Opus 4.7 (1M context) --- .../tools/transform-response-to-openai.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/apps/gateway/src/chat/tools/transform-response-to-openai.ts b/apps/gateway/src/chat/tools/transform-response-to-openai.ts index cfdd19649b..9c8ec3f0b3 100644 --- a/apps/gateway/src/chat/tools/transform-response-to-openai.ts +++ b/apps/gateway/src/chat/tools/transform-response-to-openai.ts @@ -578,6 +578,54 @@ export function transformResponseToOpenai( case "mistral": case "novita": case "openai": { + // Handle OpenAI image generation responses (e.g. gpt-image-2) + // Format: { created: number, data: [{ b64_json?: string, url?: string }], usage?: {...} } + if ( + usedProvider === "openai" && + json.data && + Array.isArray(json.data) && + !json.choices && + !json.output + ) { + transformedResponse = { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion", + created: json.created ?? Math.floor(Date.now() / 1000), + model: `${usedProvider}/${baseModelName}`, + choices: [ + { + index: 0, + message: { + role: "assistant", + content: content, + ...(images && images.length > 0 && { images }), + }, + finish_reason: finishReason ?? "stop", + }, + ], + usage: buildUsageObject( + promptTokens, + completionTokens, + totalTokens, + reasoningTokens, + cachedTokens, + costs, + showUpgradeMessage, + cacheCreationTokens, + ), + metadata: buildMetadata( + requestedModel, + requestedProvider, + baseModelName, + usedProvider, + usedModel, + requestId, + routing, + usedRegion, + ), + }; + break; + } // Handle OpenAI responses format transformation to chat completions format if (json.output && Array.isArray(json.output)) { // This is from the responses endpoint - transform to chat completions format From 1818a3a6db53139c66a481caad9ce4b16c49d0b8 Mon Sep 17 00:00:00 2001 From: smakosh Date: Thu, 23 Apr 2026 22:37:52 +0200 Subject: [PATCH 4/5] fix(gpt-image): multipart form-data for openai edits Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/gateway/src/chat/chat.ts | 18 +++-- .../chat/tools/resolve-provider-context.ts | 4 +- packages/actions/src/prepare-request-body.ts | 78 +++++++++++++++++-- 3 files changed, 87 insertions(+), 13 deletions(-) diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 6ff397aa05..27646eff4b 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -3834,7 +3834,7 @@ chat.openapi(completions, async (c) => { } } - let requestBody: ProviderRequestBody = await prepareRequestBody( + let requestBody: ProviderRequestBody | FormData = await prepareRequestBody( usedProvider, upstreamModelName, messages as BaseMessage[], @@ -3863,6 +3863,7 @@ chat.openapi(completions, async (c) => { // Validate effective max_tokens value after prepareRequestBody if ( + !(requestBody instanceof FormData) && hasMaxTokens(requestBody) && requestBody.max_tokens !== undefined && finalModelInfo @@ -3892,17 +3893,19 @@ chat.openapi(completions, async (c) => { isImageGeneration && usedProvider === "xai" && url && + !(requestBody instanceof FormData) && ("image" in requestBody || "images" in requestBody) ) { url = url.replace("/v1/images/generations", "/v1/images/edits"); } - // Switch OpenAI image generation endpoint to /edits when input images are present + // Switch OpenAI image generation endpoint to /edits when input images are present. + // prepareRequestBody returns a FormData (multipart/form-data) only for this edits flow. if ( isImageGeneration && usedProvider === "openai" && url && - "image" in requestBody + requestBody instanceof FormData ) { url = url.replace("/v1/images/generations", "/v1/images/edits"); } @@ -7821,7 +7824,9 @@ chat.openapi(completions, async (c) => { requestId, webSearchEnabled: !!webSearchTool, }); - headers["Content-Type"] = "application/json"; + if (!(requestBody instanceof FormData)) { + headers["Content-Type"] = "application/json"; + } // Add effort beta header for Anthropic if effort parameter is specified if (usedProvider === "anthropic" && effort !== undefined) { @@ -7851,7 +7856,10 @@ chat.openapi(completions, async (c) => { res = await fetch(url, { method: "POST", headers, - body: JSON.stringify(requestBody), + body: + requestBody instanceof FormData + ? requestBody + : JSON.stringify(requestBody), signal: fetchSignal, }); } catch (error) { diff --git a/apps/gateway/src/chat/tools/resolve-provider-context.ts b/apps/gateway/src/chat/tools/resolve-provider-context.ts index e4251b5c9f..d7a40e26d0 100644 --- a/apps/gateway/src/chat/tools/resolve-provider-context.ts +++ b/apps/gateway/src/chat/tools/resolve-provider-context.ts @@ -43,7 +43,7 @@ export interface ProviderContext { configIndex: number; envVarName: string | undefined; url: string; - requestBody: ProviderRequestBody; + requestBody: ProviderRequestBody | FormData; useResponsesApi: boolean; requestCanBeCanceled: boolean; isImageGeneration: boolean; @@ -365,7 +365,7 @@ export async function resolveProviderContext( providers.find((p) => p.id === usedProvider)?.cancellation === true; // --- Request body preparation --- - const requestBody: ProviderRequestBody = await prepareRequestBody( + const requestBody: ProviderRequestBody | FormData = await prepareRequestBody( usedProvider as Provider, upstreamModelName, options.messages as BaseMessage[], diff --git a/packages/actions/src/prepare-request-body.ts b/packages/actions/src/prepare-request-body.ts index e66fee1ea4..a257167eae 100644 --- a/packages/actions/src/prepare-request-body.ts +++ b/packages/actions/src/prepare-request-body.ts @@ -18,6 +18,55 @@ import { import { transformAnthropicMessages } from "./transform-anthropic-messages.js"; import { transformGoogleMessages } from "./transform-google-messages.js"; +interface OpenAIImageRequest { + model: string; + prompt: string; + size?: string; + n?: number; + image?: string | string[]; +} + +/** + * Decode an input image URL (https URL or data URL) into a Blob for multipart upload. + * Returns the Blob with the mime type and a filename with a matching extension. + */ +async function fetchImageAsBlob( + url: string, + index: number, +): Promise<{ blob: Blob; filename: string }> { + const dataUrlMatch = url.match(/^data:([^;,]+)(?:;[^,]*)?,(.*)$/); + if (dataUrlMatch) { + const mimeType = dataUrlMatch[1] || "image/png"; + const payload = dataUrlMatch[2] ?? ""; + const isBase64 = /;base64,/i.test(url.slice(0, url.indexOf(",") + 1)); + const raw = isBase64 + ? Buffer.from(payload, "base64") + : Buffer.from(decodeURIComponent(payload), "utf-8"); + const buffer = new ArrayBuffer(raw.byteLength); + new Uint8Array(buffer).set(raw); + const ext = mimeType.split("/")[1]?.split("+")[0] ?? "png"; + return { + blob: new Blob([buffer], { type: mimeType }), + filename: `image-${index}.${ext}`, + }; + } + + const response = await fetch(url); + if (!response.ok) { + throw new Error( + `Failed to fetch image ${url}: ${response.status} ${response.statusText}`, + ); + } + const mimeType = + response.headers.get("content-type")?.split(";")[0]?.trim() || "image/png"; + const buffer = await response.arrayBuffer(); + const ext = mimeType.split("/")[1]?.split("+")[0] ?? "png"; + return { + blob: new Blob([buffer], { type: mimeType }), + filename: `image-${index}.${ext}`, + }; +} + /** * Type guard to check if a tool is a function tool */ @@ -578,7 +627,7 @@ export async function prepareRequestBody( webSearchTool?: WebSearchTool, reasoning_max_tokens?: number, useResponsesApi?: boolean, -): Promise { +): Promise { // Handle OpenAI image generation models (e.g. gpt-image-2) if (imageGenerations && usedProvider === "openai") { // Extract prompt and image URLs from last user message @@ -643,7 +692,7 @@ export async function prepareRequestBody( } } - const openaiImageRequest: any = { + const openaiImageRequest: OpenAIImageRequest = { model: usedModel, prompt, ...(openaiSize && { size: openaiSize }), @@ -651,12 +700,29 @@ export async function prepareRequestBody( }; if (imageUrls.length > 0) { - // For edits flow (handled by URL swap in chat.ts) - openaiImageRequest.image = - imageUrls.length === 1 ? imageUrls[0] : imageUrls; + // Edits flow: chat.ts swaps the URL to /v1/images/edits, which requires + // multipart/form-data with binary image files rather than JSON. + const formData = new FormData(); + formData.append("model", openaiImageRequest.model); + formData.append("prompt", openaiImageRequest.prompt); + if (openaiImageRequest.size) { + formData.append("size", openaiImageRequest.size); + } + if (openaiImageRequest.n !== undefined) { + formData.append("n", String(openaiImageRequest.n)); + } + + const decoded = await Promise.all( + imageUrls.map((url, index) => fetchImageAsBlob(url, index)), + ); + const fieldName = decoded.length === 1 ? "image" : "image[]"; + for (const { blob, filename } of decoded) { + formData.append(fieldName, blob, filename); + } + return formData; } - return openaiImageRequest; + return openaiImageRequest as unknown as ProviderRequestBody; } // Handle xAI image generation models From 9820f15cb71fe86bdf1e002c83f80339a1077815 Mon Sep 17 00:00:00 2001 From: smakosh Date: Thu, 23 Apr 2026 22:56:50 +0200 Subject: [PATCH 5/5] fix(gpt-image): narrow FormData before max_tokens guard Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/gateway/src/chat/tools/resolve-provider-context.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/gateway/src/chat/tools/resolve-provider-context.ts b/apps/gateway/src/chat/tools/resolve-provider-context.ts index d7a40e26d0..8f0a56ac25 100644 --- a/apps/gateway/src/chat/tools/resolve-provider-context.ts +++ b/apps/gateway/src/chat/tools/resolve-provider-context.ts @@ -394,6 +394,7 @@ export async function resolveProviderContext( // Post-validation of max_tokens in request body if ( + !(requestBody instanceof FormData) && hasMaxTokens(requestBody) && requestBody.max_tokens !== undefined && providerMappingForSelected