feat(video): first/last frame for Seedance 2.0

apps/api/src/routes/video.ts

@@ -5,6 +5,7 @@ import { z } from "zod";
 import { userHasOrganizationAccess } from "@/utils/authorization.js";
 
 import { db } from "@llmgateway/db";
+import { buildSignedGatewayVideoLogContentUrl } from "@llmgateway/shared/video-access";
 
 import type { ServerTypes } from "@/vars.js";
 
@@ -42,6 +43,15 @@ const videoJobResponseSchema = z.object({
 			details: z.unknown().optional(),
 		})
 		.nullable(),
+	content: z
+		.array(
+			z.object({
+				type: z.literal("video"),
+				url: z.string(),
+				mime_type: z.string().nullable().optional(),
+			}),
+		)
+		.optional(),
 });
 
 const getVideoStatus = createRoute({
@@ -92,6 +102,29 @@ video.openapi(getVideoStatus, async (c) => {
 		throw new HTTPException(404, { message: "Video not found" });
 	}
 
+	let content:
+		| { type: "video"; url: string; mime_type?: string | null }[]
+		| undefined;
+	if (job.status === "completed") {
+		const log = await db.query.log.findFirst({
+			where: {
+				requestId: { eq: job.requestId },
+			},
+			columns: {
+				id: true,
+			},
+		});
+		if (log) {
+			content = [
+				{
+					type: "video" as const,
+					url: buildSignedGatewayVideoLogContentUrl(log.id),
+					mime_type: job.contentType ?? null,
+				},
+			];
+		}
+	}
+
 	return c.json(
 		{
 			id: job.id,
@@ -107,6 +140,7 @@ video.openapi(getVideoStatus, async (c) => {
 			completed_at: toUnixTimestamp(job.completedAt),
 			expires_at: toUnixTimestamp(job.expiresAt),
 			error: job.error ?? null,
+			content,
 		},
 		200,
 	);

apps/docs/content/features/video-generation.mdx

@@ -37,8 +37,8 @@ LLMGateway currently supports a focused subset of the OpenAI video API.
 | `seconds`          | number  | yes      | Duration in seconds. Supported values depend on the model (see below)                                                      |
 | `size`             | string  | no       | `widthxheight`, limited to the sizes supported by the selected model and provider                                          |
 | `audio`            | boolean | no       | Whether to include audio in the output (default `true`). Only honored when the model supports both audio and silent output |
-| `image`            | object  | no       | Optional first frame for image-to-video generation                                                                         |
-| `last_frame`       | object  | no       | Optional ending frame when `image` is provided                                                                             |
+| `image`            | object  | no       | Optional first frame for image-to-video generation (see first/last frame inputs below)                                     |
+| `last_frame`       | object  | no       | Optional ending frame when `image` is provided (see first/last frame inputs below)                                         |
 | `reference_images` | array   | no       | One to three provider-specific image inputs                                                                                |
 | `input_reference`  | object  | no       | Alias for one or more `reference_images`                                                                                   |
 | `reference_videos` | array   | no       | One to three reference video HTTPS URLs (Seedance 2.0 only, see below)                                                     |
@@ -56,6 +56,37 @@ LLMGateway currently supports a focused subset of the OpenAI video API.
 
 Requests return `400` when the selected provider cannot serve the requested `size` or `seconds`. Seedance derives `aspect_ratio` from the requested `size` (16:9 for landscape, 9:16 for portrait).
 
+### First/last frame inputs
+
+Frame inputs interpolate a video between a starting frame and an optional ending frame. You provide the first frame as `image` and, optionally, the ending frame as `last_frame`. The gateway tags each one with the correct role for the provider, so you don't set roles yourself.
+
+| Field        | Required           | Accepted input                   | Available on                                                                             |
+| ------------ | ------------------ | -------------------------------- | ---------------------------------------------------------------------------------------- |
+| `image`      | for frame mode     | HTTPS URL **or** base64 data URL | **Seedance 2.0** (`bytedance`), Veo 3.1 (`google-vertex`, `avalanche`), `minimax`, `xai` |
+| `last_frame` | no (needs `image`) | HTTPS URL **or** base64 data URL | **Seedance 2.0** (`bytedance`), Veo 3.1 (`google-vertex`, `avalanche`)                   |
+
+#### Rules and limits
+
+- **Seedance scope.** Frame inputs are supported on **Seedance 2.0** and **Seedance 2.0 Fast** (`seedance-2-0`, `seedance-2-0-fast`). Sending `image`/`last_frame` to Seedance 1.5 Pro or any other ByteDance model returns a `400`.
+- **`last_frame` requires `image`.** Providing `last_frame` without `image` returns a `400`.
+- **Not combinable with references.** First/last frame inputs (`image`, `last_frame`) cannot be combined with reference inputs (`reference_images`, `input_reference`, `reference_videos`, `reference_audios`).
+
+#### Example (Seedance 2.0)
+
+```bash
+curl -X POST "https://api.llmgateway.io/v1/videos" \
+  -H "Authorization: Bearer $LLM_GATEWAY_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "seedance-2-0",
+    "prompt": "Morph smoothly from the first frame into the last frame",
+    "seconds": 5,
+    "size": "1280x720",
+    "image": { "image_url": "https://example.com/first-frame.png" },
+    "last_frame": { "image_url": "https://example.com/last-frame.png" }
+  }'
+```
+
 ### Reference-guided generation (Seedance 2.0)
 
 Seedance 2.0 (`seedance-2-0`, `seedance-2-0-fast`) can generate a video that is guided by reference **images**, **videos**, and **audio** — sometimes called omni-reference. You attach references as top-level fields in the same `POST /v1/videos` payload; the gateway forwards each one to the provider tagged with the correct role, so you don't set roles yourself.

apps/gateway/src/test-utils/mock-openai-server.ts

@@ -1757,6 +1757,75 @@ mockOpenAIServer.post("/api/v1/jobs/createTask", async (c) => {
 	});
 });
 
+mockOpenAIServer.post("/contents/generations/tasks", async (c) => {
+	const body = await c.req.json();
+	const content = Array.isArray(body.content) ? body.content : [];
+	const promptItem = content.find(
+		(item: unknown): item is { text: string } =>
+			!!item &&
+			typeof item === "object" &&
+			(item as Record<string, unknown>).type === "text",
+	);
+	const prompt =
+		promptItem && typeof promptItem.text === "string" ? promptItem.text : "";
+	const statusTrigger = extractStatusCodeTrigger(prompt);
+	if (statusTrigger) {
+		c.status(statusTrigger.statusCode as any);
+		return c.json(statusTrigger.errorResponse);
+	}
+
+	const parseFrameByRole = (role: string) => {
+		const item = content.find(
+			(entry: unknown): entry is Record<string, unknown> =>
+				!!entry &&
+				typeof entry === "object" &&
+				(entry as Record<string, unknown>).role === role,
+		);
+		const url =
+			item &&
+			typeof item.image_url === "object" &&
+			item.image_url !== null &&
+			typeof (item.image_url as Record<string, unknown>).url === "string"
+				? ((item.image_url as Record<string, unknown>).url as string)
+				: undefined;
+		if (!url) {
+			return undefined;
+		}
+		const match = url.match(/^data:([^;]+);base64,(.*)$/);
+		if (!match) {
+			return undefined;
+		}
+		return {
+			mimeType: match[1],
+			bytesBase64Encoded: match[2],
+		};
+	};
+
+	videoCounter++;
+	const id = `bytedance_task_${videoCounter}`;
+	const job: MockVideoJobState = {
+		id,
+		object: "video",
+		model: typeof body.model === "string" ? body.model : "seedance-2-0",
+		status: "queued",
+		progress: 0,
+		firstFrame: parseFrameByRole("first_frame"),
+		lastFrame: parseFrameByRole("last_frame"),
+		duration: typeof body.duration === "number" ? body.duration : undefined,
+		created_at: Math.floor(Date.now() / 1000),
+		completed_at: null,
+		expires_at: null,
+		error: null,
+	};
+
+	videoJobs.set(id, job);
+
+	return c.json({
+		id,
+		status: "queued",
+	});
+});
+
 mockOpenAIServer.post("/api/file-base64-upload", async (c) => {
 	const authHeader = c.req.header("Authorization");
 	if (!authHeader?.startsWith("Bearer ")) {

apps/gateway/src/videos/videos.spec.ts

@@ -1015,6 +1015,103 @@ describe("videos", () => {
 		}
 	});
 
+	test("/v1/videos forwards frame inputs to bytedance Seedance 2.0", async () => {
+		await db.insert(tables.apiKey).values({
+			id: "token-id",
+			token: "real-token",
+			projectId: "project-id",
+			description: "Test API Key",
+			createdBy: "user-id",
+		});
+
+		await db.insert(tables.providerKey).values({
+			id: "provider-key-id",
+			token: "sk-bytedance-key",
+			provider: "bytedance",
+			organizationId: "org-id",
+			baseUrl: mockServerUrl,
+		});
+
+		const createRes = await app.request("/v1/videos", {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: "Bearer real-token",
+			},
+			body: JSON.stringify({
+				model: "seedance-2-0",
+				prompt: "Morph from the first frame into the last frame",
+				size: "1280x720",
+				seconds: 5,
+				image: {
+					image_url: "data:image/png;base64,aGVsbG8=",
+				},
+				last_frame: {
+					image_url: "data:image/png;base64,d29ybGQ=",
+				},
+			}),
+		});
+
+		expect(createRes.status).toBe(200);
+		const created = await createRes.json();
+
+		const videoJob = await db.query.videoJob.findFirst({
+			where: { id: { eq: created.id } },
+		});
+		expect(videoJob?.usedProvider).toBe("bytedance");
+
+		const mockVideo = getMockVideo(videoJob!.upstreamId);
+		expect(mockVideo?.firstFrame).toEqual({
+			bytesBase64Encoded: "aGVsbG8=",
+			mimeType: "image/png",
+		});
+		expect(mockVideo?.lastFrame).toEqual({
+			bytesBase64Encoded: "d29ybGQ=",
+			mimeType: "image/png",
+		});
+	});
+
+	test("/v1/videos rejects frame inputs on non-Seedance-2.0 bytedance models", async () => {
+		await db.insert(tables.apiKey).values({
+			id: "token-id",
+			token: "real-token",
+			projectId: "project-id",
+			description: "Test API Key",
+			createdBy: "user-id",
+		});
+
+		await db.insert(tables.providerKey).values({
+			id: "provider-key-id",
+			token: "sk-bytedance-key",
+			provider: "bytedance",
+			organizationId: "org-id",
+			baseUrl: mockServerUrl,
+		});
+
+		const res = await app.request("/v1/videos", {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: "Bearer real-token",
+			},
+			body: JSON.stringify({
+				model: "bytedance/seedance-1-5-pro",
+				prompt: "Morph from the first frame into the last frame",
+				size: "1280x720",
+				seconds: 5,
+				image: {
+					image_url: "data:image/png;base64,aGVsbG8=",
+				},
+			}),
+		});
+
+		expect(res.status).toBe(400);
+		const json = await res.json();
+		expect(JSON.stringify(json)).toContain(
+			"frame inputs are currently only supported on bytedance Seedance 2.0",
+		);
+	});
+
 	test("/v1/videos forwards reference images to google-vertex preview", async () => {
 		const originalGoogleCloudProject = process.env.LLM_GOOGLE_CLOUD_PROJECT;
 		const originalGoogleVertexRegion = process.env.LLM_GOOGLE_VERTEX_REGION;

apps/gateway/src/videos/videos.ts

@@ -855,7 +855,7 @@ function isSoraVideoModelName(externalId: string): boolean {
 	return externalId === "sora-2" || externalId === "sora-2-pro";
 }
 
-function isBytedanceReferenceModel(externalId: string): boolean {
+function isBytedanceSeedance2Model(externalId: string): boolean {
 	return (
 		externalId === "dreamina-seedance-2-0-260128" ||
 		externalId === "dreamina-seedance-2-0-fast-260128"
@@ -934,17 +934,23 @@ function getVideoProviderConstraintReasons(
 		);
 	}
 
-	if (
-		!isSoraVideoModelName(provider.externalId) &&
-		inputMode === "frames" &&
-		!isGoogleVertexVideoProvider(provider.providerId) &&
-		provider.providerId !== "avalanche" &&
-		provider.providerId !== "minimax" &&
-		provider.providerId !== "xai"
-	) {
-		reasons.push(
-			"frame inputs are currently only supported through google-vertex, avalanche, minimax, or xai",
-		);
+	if (!isSoraVideoModelName(provider.externalId) && inputMode === "frames") {
+		if (provider.providerId === "bytedance") {
+			if (!isBytedanceSeedance2Model(provider.externalId)) {
+				reasons.push(
+					"frame inputs are currently only supported on bytedance Seedance 2.0 (seedance-2-0, seedance-2-0-fast)",
+				);
+			}
+		} else if (
+			!isGoogleVertexVideoProvider(provider.providerId) &&
+			provider.providerId !== "avalanche" &&
+			provider.providerId !== "minimax" &&
+			provider.providerId !== "xai"
+		) {
+			reasons.push(
+				"frame inputs are currently only supported through google-vertex, avalanche, minimax, xai, or bytedance",
+			);
+		}
 	}
 
 	if (inputMode === "reference") {
@@ -969,7 +975,7 @@ function getVideoProviderConstraintReasons(
 		}
 
 		if (provider.providerId === "bytedance") {
-			if (!isBytedanceReferenceModel(provider.externalId)) {
+			if (!isBytedanceSeedance2Model(provider.externalId)) {
 				reasons.push(
 					"reference inputs are currently only supported on bytedance Seedance 2.0 (seedance-2-0, seedance-2-0-fast)",
 				);
@@ -3159,6 +3165,7 @@ async function createBytedanceVideoJob(
 	includeAudio: boolean,
 	firstFrameInput: VideoImageInput | undefined,
 	processedFirstFrame: ProcessedVideoImageInput | null,
+	processedLastFrame: ProcessedVideoImageInput | null,
 	processedReferenceImages: ProcessedVideoImageInput[],
 	referenceVideoUrls: string[],
 	referenceAudioUrls: string[],
@@ -3185,6 +3192,16 @@ async function createBytedanceVideoJob(
 		});
 	}
 
+	if (processedLastFrame) {
+		content.push({
+			type: "image_url",
+			image_url: {
+				url: `data:${processedLastFrame.mimeType};base64,${processedLastFrame.bytesBase64Encoded}`,
+			},
+			role: "last_frame",
+		});
+	}
+
 	if (processedReferenceImages.length > 0) {
 		for (const image of processedReferenceImages) {
 			content.push({
@@ -3552,6 +3569,7 @@ async function createUpstreamVideoJob(
 				includeAudio,
 				firstFrameInput,
 				processedFirstFrame,
+				processedLastFrame,
 				processedReferenceImages,
 				referenceVideoUrls,
 				referenceAudioUrls,