diff --git a/apps/docs/content/features/video-generation.mdx b/apps/docs/content/features/video-generation.mdx index 2919b7213b..41bcca03ba 100644 --- a/apps/docs/content/features/video-generation.mdx +++ b/apps/docs/content/features/video-generation.mdx @@ -41,6 +41,8 @@ LLMGateway currently supports a focused subset of the OpenAI video API. | `last_frame` | object | no | Optional ending frame when `image` is provided | | `reference_images` | array | no | One to three provider-specific image inputs | | `input_reference` | object | no | Alias for one or more `reference_images` | +| `reference_videos` | array | no | One to three reference video HTTPS URLs (Seedance 2.0 only, see below) | +| `reference_audios` | array | no | One to three reference audio HTTPS URLs (Seedance 2.0 only, see below) | | `callback_url` | string | no | LLMGateway extension for completion webhooks | | `callback_secret` | string | no | LLMGateway extension used to sign webhook deliveries | @@ -54,6 +56,90 @@ LLMGateway currently supports a focused subset of the OpenAI video API. Requests return `400` when the selected provider cannot serve the requested `size` or `seconds`. Seedance derives `aspect_ratio` from the requested `size` (16:9 for landscape, 9:16 for portrait). +### Reference-guided generation (Seedance 2.0) + +Seedance 2.0 (`seedance-2-0`, `seedance-2-0-fast`) can generate a video that is guided by reference **images**, **videos**, and **audio** — sometimes called omni-reference. You attach references as top-level fields in the same `POST /v1/videos` payload; the gateway forwards each one to the provider tagged with the correct role, so you don't set roles yourself. + +| Reference type | Payload field | Count | Accepted input | Available on | +| -------------- | -------------------------------------------- | ----- | -------------------------------- | ---------------------------------------------------- | +| Image | `reference_images` (`input_reference` alias) | 1–3 | HTTPS URL **or** base64 data URL | Seedance 2.0, Veo 3.1 (`google-vertex`, `avalanche`) | +| Video | `reference_videos` | 1–3 | HTTPS URL only | Seedance 2.0 | +| Audio | `reference_audios` | 1–3 | HTTPS URL only | Seedance 2.0 | + +Each list item accepts either a bare URL string or an object form: + +- `reference_images`: `"https://…/subject.png"` or `{ "image_url": "https://…/subject.png" }` +- `reference_videos`: `"https://…/motion.mp4"` or `{ "video_url": "https://…/motion.mp4" }` +- `reference_audios`: `"https://…/track.mp3"` or `{ "audio_url": "https://…/track.mp3" }` + +You can mix all three reference types in one request. The `prompt` can be a light instruction (for example `"adapt this to show more detail"`) — the references drive the result. + +#### Rules and limits + +- **HTTPS only for video and audio.** `reference_videos` and `reference_audios` must be publicly reachable HTTPS URLs (the provider fetches them). base64 data URLs are rejected for video/audio; images may be HTTPS URLs or base64 data URLs. +- **Reference video resolution.** Seedance requires reference video frames to be at least ~409,600 pixels (roughly 480p or larger). Low-resolution clips such as 360p are rejected with a `400`. +- **Not combinable with frames.** Reference inputs (`reference_images`, `reference_videos`, `reference_audios`) cannot be combined with the first/last frame inputs (`image`, `last_frame`). +- **Provider scope.** Reference videos and audio are only supported on Seedance 2.0 models; sending them to other models returns a `400`. +- **Moderation still applies.** The output is subject to the provider's content moderation. Blocked generations finish as `failed` and are logged with a `content_filter` finish reason. + +#### Examples + +Reference images only (subjects / style): + +```bash +curl -X POST "https://api.llmgateway.io/v1/videos" \ + -H "Authorization: Bearer $LLM_GATEWAY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "seedance-2-0", + "prompt": "The subject walks through a neon-lit market at night", + "seconds": 5, + "size": "1280x720", + "reference_images": [ + { "image_url": "https://example.com/subject.png" }, + { "image_url": "https://example.com/style.png" } + ] + }' +``` + +Reference video only (motion / scene — let the clip drive the output): + +```bash +curl -X POST "https://api.llmgateway.io/v1/videos" \ + -H "Authorization: Bearer $LLM_GATEWAY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "seedance-2-0", + "prompt": "adapt this to show more detail", + "seconds": 5, + "size": "1280x720", + "reference_videos": ["https://example.com/reference-motion.mp4"] + }' +``` + +All three reference types combined: + +```bash +curl -X POST "https://api.llmgateway.io/v1/videos" \ + -H "Authorization: Bearer $LLM_GATEWAY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "seedance-2-0", + "prompt": "The subject performs the choreography from the reference video", + "seconds": 5, + "size": "1280x720", + "reference_images": [ + { "image_url": "https://example.com/subject.png" } + ], + "reference_videos": [ + "https://example.com/reference-motion.mp4" + ], + "reference_audios": [ + "https://example.com/reference-track.mp3" + ] + }' +``` + ### Not supported yet - multipart uploads diff --git a/apps/gateway/src/chat/tools/get-finish-reason-from-error.ts b/apps/gateway/src/chat/tools/get-finish-reason-from-error.ts index bf6c7e5892..e53162e0cc 100644 --- a/apps/gateway/src/chat/tools/get-finish-reason-from-error.ts +++ b/apps/gateway/src/chat/tools/get-finish-reason-from-error.ts @@ -1,5 +1,7 @@ import { hasInvalidProviderCredentialError } from "@/lib/provider-auth-errors.js"; +import { isContentFilterErrorText } from "@llmgateway/shared"; + /** * Determines the appropriate finish reason based on HTTP status code and error message * 5xx status codes indicate upstream provider errors @@ -31,21 +33,10 @@ export function getFinishReasonFromError( return "upstream_error"; } - // Azure OpenAI content filter (ResponsibleAIPolicyViolation) - if (errorText?.includes("ResponsibleAIPolicyViolation")) { - return "content_filter"; - } - - // ByteDance / DeepSeek provider moderation block - if (errorText?.includes("SensitiveContentDetected")) { - return "content_filter"; - } - - // Alibaba / DashScope moderation block - if ( - errorText?.includes("data_inspection_failed") || - errorText?.includes("Input data may contain inappropriate content") - ) { + // Provider content-moderation / safety blocks (Azure ResponsibleAIPolicyViolation, + // ByteDance/DeepSeek SensitiveContentDetected, Alibaba data_inspection_failed, + // Azure content management policy, OpenAI safety system rejection, etc.) + if (isContentFilterErrorText(errorText)) { return "content_filter"; } @@ -57,17 +48,6 @@ export function getFinishReasonFromError( return "content_filter"; } - // Azure OpenAI prompt-side content filter (distinct from ResponsibleAIPolicyViolation, - // which fires on the response side and includes inner_error details) - if (errorText?.includes("Microsoft's content management policy")) { - return "content_filter"; - } - - // OpenAI safety system rejection (e.g. gpt-image-2 image generation) - if (errorText?.includes("Your request was rejected by the safety system")) { - return "content_filter"; - } - // 401/403 and known provider credential payloads indicate bad provider keys. if ( statusCode === 401 || diff --git a/apps/gateway/src/videos/videos.spec.ts b/apps/gateway/src/videos/videos.spec.ts index 75794cf862..3b0135e7c1 100644 --- a/apps/gateway/src/videos/videos.spec.ts +++ b/apps/gateway/src/videos/videos.spec.ts @@ -171,6 +171,201 @@ describe("videos", () => { ); }); + test("/v1/videos rejects non-https reference videos", async () => { + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + const res = await app.request("/v1/videos", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "seedance-2-0", + prompt: "Reproduce the camera move from the reference clip", + size: "1280x720", + seconds: 5, + reference_videos: ["http://example.com/reference-motion.mp4"], + }), + }); + + expect(res.status).toBe(400); + const json = await res.json(); + expect(JSON.stringify(json)).toContain("reference_videos"); + }); + + test("/v1/videos rejects combining frames with reference videos", async () => { + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + const res = await app.request("/v1/videos", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "seedance-2-0", + prompt: "Blend these inputs", + size: "1280x720", + seconds: 5, + image: { image_url: "data:image/png;base64,aGVsbG8=" }, + reference_videos: ["https://example.com/reference-motion.mp4"], + }), + }); + + expect(res.status).toBe(400); + const json = await res.json(); + expect(JSON.stringify(json)).toContain("Frame inputs"); + }); + + test("/v1/videos rejects reference videos on non-bytedance models", async () => { + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + await db.insert(tables.providerKey).values({ + id: "provider-key-id", + token: "sk-test-key", + provider: "avalanche", + organizationId: "org-id", + baseUrl: mockServerUrl, + }); + + const res = await app.request("/v1/videos", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "avalanche/veo-3.1-fast-generate-preview", + prompt: "Reproduce this motion", + size: "1920x1080", + seconds: 8, + reference_videos: ["https://example.com/reference-motion.mp4"], + }), + }); + + expect(res.status).toBe(400); + const json = await res.json(); + expect(JSON.stringify(json)).toContain( + "reference videos are currently only supported on bytedance", + ); + }); + + test("/v1/videos rejects non-https reference audios", async () => { + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + const res = await app.request("/v1/videos", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "seedance-2-0", + prompt: "Align the motion to the reference track", + size: "1280x720", + seconds: 5, + reference_audios: ["http://example.com/reference-track.mp3"], + }), + }); + + expect(res.status).toBe(400); + const json = await res.json(); + expect(JSON.stringify(json)).toContain("reference_audios"); + }); + + test("/v1/videos rejects reference audio on non-bytedance models", async () => { + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + await db.insert(tables.providerKey).values({ + id: "provider-key-id", + token: "sk-test-key", + provider: "avalanche", + organizationId: "org-id", + baseUrl: mockServerUrl, + }); + + const res = await app.request("/v1/videos", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "avalanche/veo-3.1-fast-generate-preview", + prompt: "Align this motion", + size: "1920x1080", + seconds: 8, + reference_audios: ["https://example.com/reference-track.mp3"], + }), + }); + + expect(res.status).toBe(400); + const json = await res.json(); + expect(JSON.stringify(json)).toContain( + "reference audio is currently only supported on bytedance", + ); + }); + + test("/v1/videos restricts reference inputs to Seedance 2.0 models", async () => { + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + + const res = await app.request("/v1/videos", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "bytedance/seedance-1-5-pro", + prompt: "Reproduce this motion", + size: "1280x720", + seconds: 5, + reference_videos: ["https://example.com/reference-motion.mp4"], + }), + }); + + expect(res.status).toBe(400); + const json = await res.json(); + expect(JSON.stringify(json)).toContain("Seedance 2.0"); + }); + test("/v1/videos uses routing metrics to pick the best eligible provider", async () => { const originalGoogleCloudProject = process.env.LLM_GOOGLE_CLOUD_PROJECT; const originalRuntimeGoogleCloudProject = process.env.GOOGLE_CLOUD_PROJECT; diff --git a/apps/gateway/src/videos/videos.ts b/apps/gateway/src/videos/videos.ts index d31d3734cb..e7b838a3d7 100644 --- a/apps/gateway/src/videos/videos.ts +++ b/apps/gateway/src/videos/videos.ts @@ -161,6 +161,60 @@ const videoImageInputSchema = z const videoReferenceImagesSchema = z.array(videoImageInputSchema).min(1).max(3); +const referenceVideoUrlSchema = z + .string() + .url() + .refine((value) => /^https:\/\//i.test(value), { + message: "Reference video URL must be an HTTPS URL", + }); + +const videoReferenceVideoInputSchema = z + .union([ + referenceVideoUrlSchema, + z.object({ + video_url: referenceVideoUrlSchema, + }), + ]) + .openapi({ + description: + "Reference video input for omni-reference video generation. Must be a publicly reachable HTTPS URL; base64 data URLs are not supported for videos.", + example: { + video_url: "https://example.com/reference-motion.mp4", + }, + }); + +const videoReferenceVideosSchema = z + .array(videoReferenceVideoInputSchema) + .min(1) + .max(3); + +const referenceAudioUrlSchema = z + .string() + .url() + .refine((value) => /^https:\/\//i.test(value), { + message: "Reference audio URL must be an HTTPS URL", + }); + +const videoReferenceAudioInputSchema = z + .union([ + referenceAudioUrlSchema, + z.object({ + audio_url: referenceAudioUrlSchema, + }), + ]) + .openapi({ + description: + "Reference audio input for omni-reference video generation. Must be a publicly reachable HTTPS URL; base64 data URLs are not supported for audio.", + example: { + audio_url: "https://example.com/reference-track.mp3", + }, + }); + +const videoReferenceAudiosSchema = z + .array(videoReferenceAudioInputSchema) + .min(1) + .max(3); + const createVideoRequestSchema = z .object({ model: z.string().default("veo-3.1-generate-preview").openapi({ @@ -223,6 +277,24 @@ const createVideoRequestSchema = z }, ], }), + reference_videos: videoReferenceVideosSchema.optional().openapi({ + description: + "One to three reference videos (HTTPS URLs) for omni-reference video generation. Currently only supported on ByteDance Seedance 2.0 models and can be combined with reference_images.", + example: [ + { + video_url: "https://example.com/reference-motion.mp4", + }, + ], + }), + reference_audios: videoReferenceAudiosSchema.optional().openapi({ + description: + "One to three reference audio clips (HTTPS URLs) for omni-reference video generation. Currently only supported on ByteDance Seedance 2.0 models and can be combined with reference_images and reference_videos.", + example: [ + { + audio_url: "https://example.com/reference-track.mp3", + }, + ], + }), }) .superRefine((value, ctx) => { const hasCallbackUrl = value.callback_url !== undefined; @@ -258,7 +330,9 @@ const createVideoRequestSchema = z value.image !== undefined || value.last_frame !== undefined; const hasReferenceInput = value.reference_images !== undefined || - value.input_reference !== undefined; + value.input_reference !== undefined || + value.reference_videos !== undefined || + value.reference_audios !== undefined; if (value.last_frame !== undefined && value.image === undefined) { ctx.addIssue({ @@ -700,6 +774,13 @@ function isSoraVideoModelName(externalId: string): boolean { return externalId === "sora-2" || externalId === "sora-2-pro"; } +function isBytedanceReferenceModel(externalId: string): boolean { + return ( + externalId === "dreamina-seedance-2-0-260128" || + externalId === "dreamina-seedance-2-0-fast-260128" + ); +} + function isGoogleVertexVideoProvider(providerId: string): boolean { return providerId === "google-vertex"; } @@ -710,6 +791,8 @@ function getVideoProviderConstraintReasons( videoDurationSeconds: number, inputMode: VideoInputMode, inputImageCount: number, + referenceVideoCount: number, + referenceAudioCount: number, includeAudio: boolean, ): string[] { const reasons: string[] = []; @@ -783,6 +866,16 @@ function getVideoProviderConstraintReasons( if (inputMode === "reference") { if (isSoraVideoModelName(provider.externalId)) { + if (referenceVideoCount > 0) { + reasons.push( + "Sora models do not support reference videos. Use reference_images with exactly one image.", + ); + } + if (referenceAudioCount > 0) { + reasons.push( + "Sora models do not support reference audio. Use reference_images with exactly one image.", + ); + } if (inputImageCount !== 1) { reasons.push( "Sora reference-image video generation supports exactly 1 input image", @@ -792,6 +885,28 @@ function getVideoProviderConstraintReasons( return reasons; } + if (provider.providerId === "bytedance") { + if (!isBytedanceReferenceModel(provider.externalId)) { + reasons.push( + "reference inputs are currently only supported on bytedance Seedance 2.0 (seedance-2-0, seedance-2-0-fast)", + ); + } + + return reasons; + } + + if (referenceVideoCount > 0) { + reasons.push( + "reference videos are currently only supported on bytedance Seedance 2.0 models", + ); + } + + if (referenceAudioCount > 0) { + reasons.push( + "reference audio is currently only supported on bytedance Seedance 2.0 models", + ); + } + if (isGoogleVertexVideoProvider(provider.providerId)) { if (provider.externalId !== "veo-3.1-generate-001") { reasons.push( @@ -827,6 +942,8 @@ function formatVideoProviderConstraintSummary( videoDurationSeconds: number, inputMode: VideoInputMode, inputImageCount: number, + referenceVideoCount: number, + referenceAudioCount: number, includeAudio: boolean, ): string { const providerSummaries = providers.map((provider) => { @@ -836,6 +953,8 @@ function formatVideoProviderConstraintSummary( videoDurationSeconds, inputMode, inputImageCount, + referenceVideoCount, + referenceAudioCount, includeAudio, ); return `${provider.providerId}: ${reasons.join("; ")}`; @@ -856,6 +975,8 @@ function getEligibleVideoProviderMappings( videoDurationSeconds: number, inputMode: VideoInputMode, inputImageCount: number, + referenceVideoCount: number, + referenceAudioCount: number, includeAudio: boolean, ): ProviderModelMapping[] { const now = new Date(); @@ -886,6 +1007,8 @@ function getEligibleVideoProviderMappings( videoDurationSeconds, inputMode, inputImageCount, + referenceVideoCount, + referenceAudioCount, includeAudio, ).length === 0 ); @@ -900,6 +1023,8 @@ function getEligibleVideoProviderMappings( videoDurationSeconds, inputMode, inputImageCount, + referenceVideoCount, + referenceAudioCount, includeAudio, ), }); @@ -1370,6 +1495,8 @@ async function resolveVideoExecution( videoDurationSeconds: number, inputMode: VideoInputMode, inputImageCount: number, + referenceVideoCount: number, + referenceAudioCount: number, includeAudio: boolean, project: InferSelectModel, organizationId: string, @@ -1397,6 +1524,8 @@ async function resolveVideoExecution( videoDurationSeconds, inputMode, inputImageCount, + referenceVideoCount, + referenceAudioCount, includeAudio, ); const configuredEligibleMappings: ProviderModelMapping[] = []; @@ -1446,6 +1575,8 @@ async function resolveVideoExecution( videoDurationSeconds, inputMode, inputImageCount, + referenceVideoCount, + referenceAudioCount, includeAudio, ), }); @@ -2840,6 +2971,8 @@ async function createBytedanceVideoJob( firstFrameInput: VideoImageInput | undefined, processedFirstFrame: ProcessedVideoImageInput | null, processedReferenceImages: ProcessedVideoImageInput[], + referenceVideoUrls: string[], + referenceAudioUrls: string[], ): Promise<{ upstreamId: string; upstreamRequest: Record; @@ -2859,6 +2992,7 @@ async function createBytedanceVideoJob( image_url: { url: `data:${processedFirstFrame.mimeType};base64,${processedFirstFrame.bytesBase64Encoded}`, }, + role: "first_frame", }); } @@ -2869,10 +3003,31 @@ async function createBytedanceVideoJob( image_url: { url: `data:${image.mimeType};base64,${image.bytesBase64Encoded}`, }, + role: "reference_image", }); } } + for (const referenceVideoUrl of referenceVideoUrls) { + content.push({ + type: "video_url", + video_url: { + url: referenceVideoUrl, + }, + role: "reference_video", + }); + } + + for (const referenceAudioUrl of referenceAudioUrls) { + content.push({ + type: "audio_url", + audio_url: { + url: referenceAudioUrl, + }, + role: "reference_audio", + }); + } + const isDreaminaModel = upstreamModelName.startsWith("dreamina-"); const upstreamRequest: Record = { @@ -2934,6 +3089,8 @@ async function createUpstreamVideoJob( firstFrameInput: VideoImageInput | undefined, lastFrameInput: VideoImageInput | undefined, referenceImageInputs: VideoImageInput[], + referenceVideoUrls: string[], + referenceAudioUrls: string[], processedFirstFrame: ProcessedVideoImageInput | null, processedLastFrame: ProcessedVideoImageInput | null, processedReferenceImages: ProcessedVideoImageInput[], @@ -2986,6 +3143,8 @@ async function createUpstreamVideoJob( firstFrameInput, processedFirstFrame, processedReferenceImages, + referenceVideoUrls, + referenceAudioUrls, ); case "google-vertex": return await createGoogleVertexVideoJob( @@ -3048,7 +3207,9 @@ function getVideoInputMode( if ( request.reference_images !== undefined || - request.input_reference !== undefined + request.input_reference !== undefined || + request.reference_videos !== undefined || + request.reference_audios !== undefined ) { return "reference"; } @@ -3056,6 +3217,34 @@ function getVideoInputMode( return "none"; } +function getVideoReferenceVideoInputs( + request: z.infer, +): string[] { + if (!request.reference_videos) { + return []; + } + + return request.reference_videos.map((referenceVideo) => + typeof referenceVideo === "string" + ? referenceVideo + : referenceVideo.video_url, + ); +} + +function getVideoReferenceAudioInputs( + request: z.infer, +): string[] { + if (!request.reference_audios) { + return []; + } + + return request.reference_audios.map((referenceAudio) => + typeof referenceAudio === "string" + ? referenceAudio + : referenceAudio.audio_url, + ); +} + function getVideoInputImageCount( inputMode: VideoInputMode, firstFrameInput: VideoImageInput | undefined, @@ -3199,6 +3388,8 @@ videos.openapi(createVideo, async (c) => { const firstFrameInput = getVideoFirstFrameInput(request); const lastFrameInput = getVideoLastFrameInput(request); const referenceImageInputs = getVideoReferenceImageInputs(request); + const referenceVideoInputs = getVideoReferenceVideoInputs(request); + const referenceAudioInputs = getVideoReferenceAudioInputs(request); const inputMode = getVideoInputMode(request); const inputImageCount = getVideoInputImageCount( inputMode, @@ -3251,6 +3442,8 @@ videos.openapi(createVideo, async (c) => { videoDurationSeconds, inputMode, inputImageCount, + referenceVideoInputs.length, + referenceAudioInputs.length, request.audio, project, organization.id, @@ -3399,6 +3592,8 @@ videos.openapi(createVideo, async (c) => { firstFrameInput, lastFrameInput, referenceImageInputs, + referenceVideoInputs, + referenceAudioInputs, processedFirstFrame, processedLastFrameInput, processedReferenceImages, diff --git a/apps/playground/src/components/playground/video-controls.tsx b/apps/playground/src/components/playground/video-controls.tsx index 92405abd5f..34c39195fc 100644 --- a/apps/playground/src/components/playground/video-controls.tsx +++ b/apps/playground/src/components/playground/video-controls.tsx @@ -1,6 +1,14 @@ "use client"; -import { ImagePlus, Loader2, Sparkles, X } from "lucide-react"; +import { + Film, + ImagePlus, + Loader2, + Music, + Plus, + Sparkles, + X, +} from "lucide-react"; import { type Dispatch, type SetStateAction, @@ -11,6 +19,7 @@ import { } from "react"; import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; import { Select, SelectContent, @@ -43,10 +52,16 @@ interface VideoControlsProps { audioToggleDisabled: boolean; canUseFrameInputs: boolean; canUseReferenceInputs: boolean; + canUseReferenceVideoInputs: boolean; + canUseReferenceAudioInputs: boolean; frameInputs: VideoFrameInputs; setFrameInputs: Dispatch>; referenceImages: VideoInputImage[]; setReferenceImages: Dispatch>; + referenceVideos: string[]; + setReferenceVideos: Dispatch>; + referenceAudios: string[]; + setReferenceAudios: Dispatch>; supportedVideoSizes: VideoSize[]; supportedVideoDurations: VideoDuration[]; isGenerating: boolean; @@ -68,10 +83,16 @@ export function VideoControls({ audioToggleDisabled, canUseFrameInputs, canUseReferenceInputs, + canUseReferenceVideoInputs, + canUseReferenceAudioInputs, frameInputs, setFrameInputs, referenceImages, setReferenceImages, + referenceVideos, + setReferenceVideos, + referenceAudios, + setReferenceAudios, supportedVideoSizes, supportedVideoDurations, isGenerating, @@ -82,6 +103,47 @@ export function VideoControls({ const dropZoneRef = useRef(null); const [isDragging, setIsDragging] = useState(false); const [uploadTarget, setUploadTarget] = useState("frame-start"); + const [referenceVideoDraft, setReferenceVideoDraft] = useState(""); + + const addReferenceVideo = useCallback(() => { + const url = referenceVideoDraft.trim(); + if (!url || !/^https:\/\//i.test(url)) { + return; + } + setFrameInputs({ start: null, end: null }); + setReferenceVideos((prev) => + prev.length >= 3 || prev.includes(url) ? prev : [...prev, url], + ); + setReferenceVideoDraft(""); + }, [referenceVideoDraft, setFrameInputs, setReferenceVideos]); + + const removeReferenceVideo = useCallback( + (index: number) => { + setReferenceVideos((prev) => prev.filter((_, i) => i !== index)); + }, + [setReferenceVideos], + ); + + const [referenceAudioDraft, setReferenceAudioDraft] = useState(""); + + const addReferenceAudio = useCallback(() => { + const url = referenceAudioDraft.trim(); + if (!url || !/^https:\/\//i.test(url)) { + return; + } + setFrameInputs({ start: null, end: null }); + setReferenceAudios((prev) => + prev.length >= 3 || prev.includes(url) ? prev : [...prev, url], + ); + setReferenceAudioDraft(""); + }, [referenceAudioDraft, setFrameInputs, setReferenceAudios]); + + const removeReferenceAudio = useCallback( + (index: number) => { + setReferenceAudios((prev) => prev.filter((_, i) => i !== index)); + }, + [setReferenceAudios], + ); const canGenerate = prompt.trim().length > 0 && selectedModels.length > 0; const canAcceptInput = canUseFrameInputs || canUseReferenceInputs; @@ -125,6 +187,8 @@ export function VideoControls({ if (target === "frame-start") { setReferenceImages([]); + setReferenceVideos([]); + setReferenceAudios([]); setFrameInputs((prev) => ({ ...prev, start: nextImage, @@ -134,6 +198,8 @@ export function VideoControls({ if (target === "frame-end") { setReferenceImages([]); + setReferenceVideos([]); + setReferenceAudios([]); setFrameInputs((prev) => ({ ...prev, end: nextImage, @@ -162,6 +228,8 @@ export function VideoControls({ defaultUploadTarget, setFrameInputs, setReferenceImages, + setReferenceVideos, + setReferenceAudios, ], ); @@ -365,6 +433,50 @@ export function VideoControls({ ))} )} + {referenceVideos.length > 0 && ( +
+ {referenceVideos.map((url, index) => ( +
+ + Reference video {index + 1} + +
+ ))} +
+ )} + {referenceAudios.length > 0 && ( +
+ {referenceAudios.map((url, index) => ( +
+ + Reference audio {index + 1} + +
+ ))} +
+ )}