diff --git a/.changeset/generate-world.md b/.changeset/generate-world.md new file mode 100644 index 0000000000..55a74e056e --- /dev/null +++ b/.changeset/generate-world.md @@ -0,0 +1,8 @@ +--- +'@tanstack/ai': minor +'@tanstack/ai-event-client': patch +'@tanstack/ai-reactor': minor +'@tanstack/ai-fal': minor +--- + +Add `generateWorld()` and `generateLiveVideo()` for prompt-steerable sessions, plus a first-party Reactor adapter (`reactorWorld`, `reactorVideo`) and fal `falLiveVideo()` for H3 Max Director. Reactor returns a session JWT. falLiveVideo returns the WMA app id on `result.model` so the browser can call `wma(live.model)`. `generateVideo()` stays the job path that polls for a file URL. diff --git a/CLAUDE.md b/CLAUDE.md index 94bcf38ae8..34b12b1853 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -130,6 +130,7 @@ testing/ examples/ # Example applications ├── ts-react-chat/ # React chat example +├── ts-react-media/ # Image, video, live, and world generation ├── ts-solid-chat/ # Solid chat example ├── ts-vue-chat/ # Vue chat example ├── ts-svelte-chat/ # Svelte chat example diff --git a/README.md b/README.md index ffde43b164..ca4e4eccf8 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ Learn more in the - [Generations](https://tanstack.com/ai/latest/docs/media/generations) - one pattern for image generation, text-to-speech, transcription, summarization, - audio generation, and video generation. + audio generation, video generation, and world generation. - [Realtime Voice Chat](https://tanstack.com/ai/latest/docs/media/realtime-chat) - build low-latency realtime voice experiences. - [Code Mode](https://tanstack.com/ai/latest/docs/code-mode/code-mode) - let @@ -202,20 +202,21 @@ Learn more in the Official adapters include: -| Package | Use it for | -| ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------ | -| [`@tanstack/ai-openrouter`](https://tanstack.com/ai/latest/docs/adapters/openrouter) | 300+ models through one OpenRouter API, with per-request cost tracking | -| [`@tanstack/ai-openai`](https://tanstack.com/ai/latest/docs/adapters/openai) | OpenAI chat, image, video, speech, transcription, realtime, and provider tools | -| [`@tanstack/ai-anthropic`](https://tanstack.com/ai/latest/docs/adapters/anthropic) | Anthropic Claude chat, thinking, tools, structured outputs, and Vertex Claude | -| [`@tanstack/ai-gemini`](https://tanstack.com/ai/latest/docs/adapters/gemini) | Google Gemini chat, image, speech, and audio generation | -| [`@tanstack/ai-vertex`](https://tanstack.com/ai/latest/docs/adapters/vertex) | Gemini on Vertex AI with regional endpoints and Google Cloud credentials | -| [`@tanstack/ai-ollama`](https://tanstack.com/ai/latest/docs/adapters/ollama) | Local Ollama models | -| [`@tanstack/ai-grok`](https://tanstack.com/ai/latest/docs/adapters/grok) | xAI Grok chat, images, and realtime | -| [`@tanstack/ai-groq`](https://tanstack.com/ai/latest/docs/adapters/groq) | Groq low-latency inference | -| [`@tanstack/ai-elevenlabs`](https://tanstack.com/ai/latest/docs/adapters/elevenlabs) | ElevenLabs realtime voice, speech, transcription, music, and sound effects | -| [`@tanstack/ai-byteplus`](https://tanstack.com/ai/latest/docs/adapters/byteplus) | BytePlus Seed chat, Seedance video, Seedream image, and Seed Speech TTS/ASR | -| [`@tanstack/ai-fal`](https://tanstack.com/ai/latest/docs/adapters/fal) | fal.ai image, video, audio, speech, and transcription models | -| [`@tanstack/ai-cloudflare`](https://tanstack.com/ai/latest/docs/adapters/cloudflare) | Cloudflare Workers AI chat, embeddings, image, speech, transcription, and AI Gateway | +| Package | Use it for | +| ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------- | +| [`@tanstack/ai-openrouter`](https://tanstack.com/ai/latest/docs/adapters/openrouter) | 300+ models through one OpenRouter API, with per-request cost tracking | +| [`@tanstack/ai-openai`](https://tanstack.com/ai/latest/docs/adapters/openai) | OpenAI chat, image, video, speech, transcription, realtime, and provider tools | +| [`@tanstack/ai-anthropic`](https://tanstack.com/ai/latest/docs/adapters/anthropic) | Anthropic Claude chat, thinking, tools, structured outputs, and Vertex Claude | +| [`@tanstack/ai-gemini`](https://tanstack.com/ai/latest/docs/adapters/gemini) | Google Gemini chat, image, speech, and audio generation | +| [`@tanstack/ai-vertex`](https://tanstack.com/ai/latest/docs/adapters/vertex) | Gemini on Vertex AI with regional endpoints and Google Cloud credentials | +| [`@tanstack/ai-ollama`](https://tanstack.com/ai/latest/docs/adapters/ollama) | Local Ollama models | +| [`@tanstack/ai-grok`](https://tanstack.com/ai/latest/docs/adapters/grok) | xAI Grok chat, images, and realtime | +| [`@tanstack/ai-groq`](https://tanstack.com/ai/latest/docs/adapters/groq) | Groq low-latency inference | +| [`@tanstack/ai-elevenlabs`](https://tanstack.com/ai/latest/docs/adapters/elevenlabs) | ElevenLabs realtime voice, speech, transcription, music, and sound effects | +| [`@tanstack/ai-byteplus`](https://tanstack.com/ai/latest/docs/adapters/byteplus) | BytePlus Seed chat, Seedance video, Seedream image, and Seed Speech TTS/ASR | +| [`@tanstack/ai-fal`](https://tanstack.com/ai/latest/docs/adapters/fal) | fal.ai image, video, audio, speech, and transcription models | +| [`@tanstack/ai-reactor`](https://tanstack.com/ai/latest/docs/adapters/reactor) | Reactor live world and video generation (Orbis, Happy Oyster, LingBot, Helios, FastH3) | +| [`@tanstack/ai-cloudflare`](https://tanstack.com/ai/latest/docs/adapters/cloudflare) | Cloudflare Workers AI chat, embeddings, image, speech, transcription, and AI Gateway | The adapter system is tree-shakeable by activity. Import `openaiText` for chat, `openaiImage` for images, `falVideo` for video, `geminiSpeech` for TTS, and so diff --git a/docs/adapters/fal.md b/docs/adapters/fal.md index c93fbaffe0..c8fc2503ec 100644 --- a/docs/adapters/fal.md +++ b/docs/adapters/fal.md @@ -13,7 +13,7 @@ keywords: - adapter --- -The fal.ai adapter provides access to 600+ models on the fal.ai platform for image, video, audio, speech, and transcription. Unlike text-focused adapters, the fal adapter is **media-focused** — it supports `generateImage()`, `generateVideo()`, `generateAudio()`, `generateSpeech()`, and `generateTranscription()` but does not support `chat()` or tools. +The fal.ai adapter provides access to 600+ models on the fal.ai platform for image, video, live, audio, speech, and transcription. Unlike text-focused adapters, the fal adapter is **media-focused**. It supports `generateImage()`, `generateVideo()`, `generateLiveVideo()`, `generateAudio()`, `generateSpeech()`, and `generateTranscription()`. It does not support `chat()` or tools. For a full working example, see the [fal.ai example app](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). @@ -396,6 +396,12 @@ const sfx = await generateAudio({ | `fal-ai/ltx-2/text-to-video/fast` | Text-to-Video | Fast text-to-video | | `fal-ai/ltx-2/image-to-video/fast` | Image-to-Video | Fast image-to-video animation | +### Live Models + +| Model | Mode | Description | +|-------|------|-------------| +| `minimax/h3-max/director` | Live | Steerable live stream. Use `falLiveVideo()` with `generateLiveVideo()`. | + ### Text-to-Speech Models | Model | Description | @@ -473,6 +479,19 @@ Creates a fal.ai video adapter using the `FAL_KEY` environment variable or an ex **Returns:** A `FalVideoAdapter` instance for use with `generateVideo()` and `getVideoJobStatus()`. +### `falLiveVideo(model, config?)` + +Creates a fal.ai live-video adapter for H3 Max Director. `generateLiveVideo()` returns the WMA app id on `result.model` (`fal-ai/minimax-h3-max-director`). Open that id with `wma(live.model)` through a server proxy that attaches `FAL_KEY`. Do not send `live.token` as `Key` credentials. Call `allowedFalLiveVideoProxyTarget()` in the proxy so it forwards only WMA `/ice`, `/session`, `/session/heartbeat`, and Director `/ice`. + +**Parameters:** + +- `model` - `"minimax/h3-max/director"` +- `config.apiKey?` - Your fal.ai API key (falls back to `FAL_KEY` env var) + +**Returns:** A `FalLiveVideoAdapter` instance for use with `generateLiveVideo()`. + +See [Live Generation](../media/live-generation) for the browser connect step. + ### `falSpeech(model, config?)` Creates a fal.ai text-to-speech adapter. diff --git a/docs/adapters/reactor.md b/docs/adapters/reactor.md new file mode 100644 index 0000000000..f89ff19931 --- /dev/null +++ b/docs/adapters/reactor.md @@ -0,0 +1,178 @@ +--- +title: Reactor +id: reactor-adapter +description: "Generate live worlds and video with Reactor models in TanStack AI via the @tanstack/ai-reactor adapter." +keywords: + - tanstack ai + - reactor + - world generation + - live generation + - orbis + - helios + - generateWorld + - generateLiveVideo + - adapter +--- + +Reactor hosts live world and video models. You describe a scene. Then you open a session and stream video. You can steer the stream with a new prompt. + +Use `reactorWorld()` with `generateWorld()` for navigable worlds. Use `reactorVideo()` with `generateLiveVideo()` for live video models. Both mint a session token. Neither supports `chat()`. + +## Installation + + + +react: @tanstack/ai-reactor +vue: @tanstack/ai-reactor +solid: @tanstack/ai-reactor +svelte: @tanstack/ai-reactor +preact: @tanstack/ai-reactor +angular: @tanstack/ai-reactor +vanilla: @tanstack/ai-reactor +octane: @tanstack/ai-reactor + + + +Peer dependency: `@tanstack/ai`. The browser also needs `@reactor-team/js-sdk` to connect and play the stream. See [World Generation](../media/world-generation) and [Live Generation](../media/live-generation). + +A full working app is in [`examples/ts-react-media`](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). Open the World or Live tab. + +## API key + +Create a key in the [Reactor dashboard](https://www.reactor.inc/dashboard). Keys start with `rk_`. + +The example app uses [Bring Your Own Key](../advanced/byok). The browser pastes the key. The relay reads `x-byok-reactor`, then `REACTOR_API_KEY`. + +```ts +import { generateWorld } from '@tanstack/ai' +import { byokMissing, getByokKey } from '@tanstack/ai/byok/server' +import { reactorWorld } from '@tanstack/ai-reactor' +import { reactorByok } from '@tanstack/ai-reactor/byok' + +export async function POST(request: Request) { + const apiKey = getByokKey(request, reactorByok) + if (!apiKey) return byokMissing(reactorByok) + + const world = await generateWorld({ + adapter: reactorWorld('visko-orbis-stable', { apiKey }), + prompt: 'A neon cyberpunk city at night, slow aerial drift', + }) + + return Response.json({ + token: world.token, + model: world.model, + prompt: world.prompt, + expiresAt: world.expiresAt, + }) +} +``` + +The adapter mints a **session-scoped** token for that model only. Hand `world.token`, `world.model`, and `world.prompt` to the browser. Do not put the API key in the JSON body. + +## Models + +Pass a string literal so TypeScript can narrow options. + +```ts +import { reactorWorld } from '@tanstack/ai-reactor' + +const adapter = reactorWorld('visko-orbis-stable') +``` + +| Id | Connect slug | +| --- | --- | +| `visko-orbis-stable` | `reactor/visko-orbis-stable` | +| `visko-orbis-dynamic` | `reactor/visko-orbis-dynamic` | +| `happy-oyster-adventure` | `reactor/happy-oyster-adventure` | +| `happy-oyster-director` | `reactor/happy-oyster-director` | +| `lingbot-world-2` | `reactor/lingbot-world-2` | +| `lingbot` | `reactor/lingbot` | +| `helios` | `reactor/helios` | + +`world.model` is the connect slug. Pass it to `new Reactor({ modelName })`. + +## Live video + +Reactor video is a live stream, not a finished file. `generateLiveVideo()` returns a token. The browser connects, sets the prompt, and plays the track. + +```ts +import { generateLiveVideo } from '@tanstack/ai' +import { reactorVideo } from '@tanstack/ai-reactor' + +const apiKey = process.env.REACTOR_API_KEY ?? '' +const live = await generateLiveVideo({ + adapter: reactorVideo('helios', { apiKey }), + prompt: 'A neon cyberpunk city at night, slow aerial drift', +}) +``` + +Hand `live.token`, `live.model`, and `live.prompt` to the browser. Connect as shown in [Live Generation](../media/live-generation). + +| Id | Connect slug | +| --- | --- | +| `helios` | `reactor/helios` | +| `fast-h3` | `reactor/fast-h3` | +| `longlive-v2` | `reactor/longlive-v2` | +| `ltx2` | `reactor/ltx2` | + +`helios` also works with `reactorWorld()`. Pick `generateLiveVideo()` when you want a video session. Pick `generateWorld()` when you want a navigable world. + +Pass a text prompt to `generateWorld()` / `generateLiveVideo()`. Those calls mint a token. They do not send resolution or a seed image. + +After `connect`, send browser commands. LingBot starts from a seed image. Pass a `File` from ``. The SDK uploads it and returns a `FileRef`. Do not send base64. `start` still needs `set_prompt`. Send a short default, then steer after the first frame. + +```ts +import { Reactor } from '@reactor-team/js-sdk' + +const reactor = new Reactor({ modelName: 'reactor/lingbot' }) +const picker = document.querySelector('input[type="file"]') +if (!(picker instanceof HTMLInputElement)) { + throw new Error('Pick a seed image') +} +const file = picker.files?.[0] +if (file === undefined) { + throw new Error('Pick a seed image') +} +const image = await reactor.uploadFile(file) +await reactor.sendCommand('set_image', { image }) +await reactor.sendCommand('set_prompt', { prompt: 'Follow the seed image.' }) +await reactor.sendCommand('start', {}) +``` + +Helios can take the same `File` with `set_conditioning` so prompt and image land together. + +## Browser session options + +Orbis reads these on the next `start`. Keep them in client state. Send them with `sendCommand` after `connect`. They are not token-mint fields. + +```ts +import { Reactor } from '@reactor-team/js-sdk' + +const reactor = new Reactor({ modelName: 'reactor/visko-orbis-stable' }) +await reactor.sendCommand('set_resolution', { resolution: '2k' }) +await reactor.sendCommand('set_seed', { seed: 42 }) +await reactor.sendCommand('set_audio_enabled', { enabled: true }) +``` + +| Option | Command | Meaning | +| --- | --- | --- | +| `resolution` | `set_resolution` | `1080p`, `2k`, or `4k` delivery tier | +| `seed` | `set_seed` | RNG seed for the next run | +| `audioEnabled` | `set_audio_enabled` | When `false`, skip audio compute | +| `audioPrompt` | `set_audio_prompt` | Sound description, or `""` for picture-driven audio | + +## Custom endpoint + +```ts +import { reactorWorld } from '@tanstack/ai-reactor' + +const apiKey = process.env.REACTOR_API_KEY ?? '' +const adapter = reactorWorld('visko-orbis-stable', { + apiKey, + baseUrl: 'https://api.reactor.inc', +}) +``` + +## What you have now + +A server call that mints a scoped Reactor token for one world or video model. Next: connect in the browser as shown in [World Generation](../media/world-generation) or [Live Generation](../media/live-generation). diff --git a/docs/advanced/otel.md b/docs/advanced/otel.md index e49f944884..f56a27ccb7 100644 --- a/docs/advanced/otel.md +++ b/docs/advanced/otel.md @@ -215,7 +215,7 @@ otelMiddleware({ ## Beyond chat: media activities -`otelMiddleware` is not chat-only. The media activities — `generateImage`, `generateVideo`, `generateAudio`, `generateSpeech`, and `generateTranscription` — accept the **same** `otelMiddleware` value on their `middleware` option. Each is a single request → response (or submit → poll for video), so the middleware emits one span per call instead of the chat span tree: +`otelMiddleware` is not chat-only. The media activities (`generateImage`, `generateVideo`, `generateLiveVideo`, `generateWorld`, `generateAudio`, `generateSpeech`, and `generateTranscription`) accept the **same** `otelMiddleware` value on their `middleware` option. Each is a single request to response (or submit then poll for video), so the middleware emits one span per call instead of the chat span tree: ```ts import { generateImage } from '@tanstack/ai' @@ -246,6 +246,8 @@ Each media call produces one `CLIENT` span tagged with the activity's `gen_ai.op | `generateAudio` | `audio_generation` | | `generateSpeech` | `text_to_speech` | | `generateTranscription` | `transcription` | +| `generateWorld` | `world_generation` | +| `generateLiveVideo` | `live_video_generation` | | `summarize` | `summarize` | The span carries `gen_ai.system` and `gen_ai.request.model` at start and, on finish, the same `gen_ai.usage.*` / `tanstack.ai.usage.*` attributes documented above — including the `tanstack.ai.usage.billed_quantity` / `tanstack.ai.usage.billed_unit` pair for unit-billed media. When a `Meter` is supplied it records the `gen_ai.client.operation.duration` histogram, tagged per activity. For streaming video the span covers the full create → poll → complete lifecycle. Non-streaming video is two calls, so the submit itself emits no span — the run opens once the provider accepts the job, and the `getVideoJobStatus()` poll that observes a terminal state ends it. If a streaming video consumer abandons the stream before completion, the span is ended via `onAbort` (status `ERROR`, `tanstack.ai.completion.reason = cancelled`) rather than leaked. diff --git a/docs/advanced/tree-shaking.md b/docs/advanced/tree-shaking.md index 950380d713..67cdea3f81 100644 --- a/docs/advanced/tree-shaking.md +++ b/docs/advanced/tree-shaking.md @@ -39,6 +39,8 @@ import { generateImage } from '@tanstack/ai' // Image generation import { generateSpeech } from '@tanstack/ai' // Text-to-speech import { generateTranscription } from '@tanstack/ai' // Audio transcription import { generateVideo } from '@tanstack/ai' // Video generation +import { generateLiveVideo } from '@tanstack/ai' // Live video sessions +import { generateWorld } from '@tanstack/ai' // Live world sessions ``` ### Example: Chat Only @@ -274,6 +276,8 @@ Each adapter type implements a specific interface: - `TTSAdapter` - Provides `generateSpeech()` method for text-to-speech - `TranscriptionAdapter` - Provides `generateTranscription()` method for audio transcription - `VideoAdapter` - Provides `generateVideo()` method for video generation +- `LiveVideoAdapter` - Provides `createLiveVideo()` method for live video sessions +- `WorldAdapter` - Provides `createWorld()` method for live world sessions All adapters have a `kind` property that indicates their type: diff --git a/docs/advanced/typed-options.md b/docs/advanced/typed-options.md index cafa8a9041..315457d8b8 100644 --- a/docs/advanced/typed-options.md +++ b/docs/advanced/typed-options.md @@ -64,6 +64,8 @@ Each helper mirrors the activity it pairs with. Same options, same return type. | `createImageOptions` | `generateImage()` | image adapter (e.g. `openaiImage`, `falImage`) | | `createAudioOptions` | `generateAudio()` | audio adapter (e.g. `falAudio`, `geminiAudio`) | | `createVideoOptions` | `generateVideo()` / `getVideoJobStatus()` | video adapter (e.g. `falVideo`, `openaiVideo`) | +| `createLiveVideoOptions` | `generateLiveVideo()` | live adapter (e.g. `reactorVideo`, `falLiveVideo`) | +| `createWorldOptions` | `generateWorld()` | world adapter (e.g. `reactorWorld`) | | `createSpeechOptions` | `generateSpeech()` | speech adapter (e.g. `openaiSpeech`, `elevenlabsSpeech`) | | `createTranscriptionOptions` | `generateTranscription()` | transcription adapter (e.g. `openaiTranscription`, `falTranscription`) | diff --git a/docs/config.json b/docs/config.json index f2e7b13277..b82453a902 100644 --- a/docs/config.json +++ b/docs/config.json @@ -13,7 +13,7 @@ "label": "Overview", "to": "getting-started/overview", "addedAt": "2026-04-15", - "updatedAt": "2026-09-03" + "updatedAt": "2026-09-04" }, { "label": "Quick Start", @@ -223,7 +223,7 @@ "label": "OpenTelemetry", "to": "advanced/otel", "addedAt": "2026-05-08", - "updatedAt": "2026-08-08" + "updatedAt": "2026-09-04" }, { "label": "Compaction", @@ -567,7 +567,7 @@ "label": "Generations", "to": "media/generations", "addedAt": "2026-04-15", - "updatedAt": "2026-08-19" + "updatedAt": "2026-09-04" }, { "label": "Realtime Voice Chat", @@ -609,13 +609,25 @@ "label": "Video Generation", "to": "media/video-generation", "addedAt": "2026-04-15", - "updatedAt": "2026-08-31" + "updatedAt": "2026-09-04" + }, + { + "label": "Live Generation", + "to": "media/live-generation", + "addedAt": "2026-09-04", + "updatedAt": "2026-09-04" + }, + { + "label": "World Generation", + "to": "media/world-generation", + "addedAt": "2026-09-04", + "updatedAt": "2026-09-04" }, { "label": "Generation Hooks", "to": "media/generation-hooks", "addedAt": "2026-04-15", - "updatedAt": "2026-09-03" + "updatedAt": "2026-09-04" } ], "tab": "guides" @@ -912,7 +924,8 @@ { "label": "Tree-Shaking", "to": "advanced/tree-shaking", - "addedAt": "2026-04-15" + "addedAt": "2026-04-15", + "updatedAt": "2026-09-04" }, { "label": "Extend Adapter", @@ -922,7 +935,8 @@ { "label": "Typed Pre-Configured Options", "to": "advanced/typed-options", - "addedAt": "2026-05-25" + "addedAt": "2026-05-25", + "updatedAt": "2026-09-04" }, { "label": "Approval Flow Processing", @@ -1000,7 +1014,13 @@ "label": "fal.ai", "to": "adapters/fal", "addedAt": "2026-04-15", - "updatedAt": "2026-08-20" + "updatedAt": "2026-09-04" + }, + { + "label": "Reactor", + "to": "adapters/reactor", + "addedAt": "2026-09-04", + "updatedAt": "2026-09-04" }, { "label": "OpenRouter Adapter", diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index b7dba57b04..5260630643 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -125,6 +125,7 @@ With the help of adapters, TanStack AI can connect to various LLM providers. Ava - **@tanstack/ai-bedrock** - Amazon Bedrock (Claude, Nova, Llama, and more via AWS) - **@tanstack/ai-byteplus** - BytePlus (Seed chat, Seedance video, Seedream image, Seed Speech) - **@tanstack/ai-fal** - fal (image & video generation) +- **@tanstack/ai-reactor** - Reactor (live world and video generation: Orbis, Happy Oyster, LingBot, Helios, FastH3) - **@tanstack/ai-llmgateway** - LLM Gateway (hundreds of models via one OpenAI-compatible endpoint, self-hostable) - **@tanstack/ai-cloudflare** - Cloudflare Workers AI (chat, embeddings, image, speech, transcription) and AI Gateway routing - **@tanstack/ai-lovable** - Lovable AI Gateway (Google and OpenAI chat, image, video, embeddings, and speech via one project key) diff --git a/docs/media/generation-hooks.md b/docs/media/generation-hooks.md index bd42e01712..7b231fcfb5 100644 --- a/docs/media/generation-hooks.md +++ b/docs/media/generation-hooks.md @@ -67,7 +67,7 @@ export async function POST(req: Request) { } ``` -The same pattern applies to all generation types -- swap `generateImage` for `generateSpeech`, `generateTranscription`, `summarize`, or `generateVideo`. See the individual media guides for server-side details. +The same pattern applies to image, speech, transcription, summarize, and video. Live and world generation mint a session token. Connect in the browser. See [Live Generation](./live-generation) and [World Generation](./world-generation). ## useGenerateImage diff --git a/docs/media/generations.md b/docs/media/generations.md index ccc2696910..57c7bee893 100644 --- a/docs/media/generations.md +++ b/docs/media/generations.md @@ -15,10 +15,12 @@ keywords: # Generations -You want an image, some speech, a transcript or a video, not a conversation. Every one +You want an image, some speech, a transcript, a video, or a live stream, not a conversation. Every one of those is a **generation**: one request, one result. They all share the same shape, so learning one teaches you the rest. +For a live, prompt-steerable stream that does not end as a file, see [Live Generation](./live-generation) or [World Generation](./world-generation). + ## The fastest path A server route that streams the result: @@ -87,8 +89,10 @@ streaming is handled for you. All three are written out in | Transcription | `generateTranscription()` | `useTranscription()` | [Transcription](./transcription) | | Summarization | `summarize()` | `useSummarize()` | - | | Video generation | `generateVideo()` | `useGenerateVideo()` | [Video Generation](./video-generation) | +| Live generation | `generateLiveVideo()` | - | [Live Generation](./live-generation) | +| World generation | `generateWorld()` | - | [World Generation](./world-generation) | -> **Note:** Video generation uses a jobs/polling architecture. The `useGenerateVideo` hook additionally exposes `jobId`, `videoStatus`, `onJobCreated`, and `onStatusUpdate` for tracking the polling lifecycle. See the [Video Generation](./video-generation) guide for details. +> **Note:** Video adapters use a jobs/polling architecture. The `useGenerateVideo` hook additionally exposes `jobId`, `videoStatus`, `onJobCreated`, and `onStatusUpdate` for tracking the polling lifecycle. Live and world sessions mint a token. There is no download URL. ## Advanced diff --git a/docs/media/live-generation.md b/docs/media/live-generation.md new file mode 100644 index 0000000000..4cccee7139 --- /dev/null +++ b/docs/media/live-generation.md @@ -0,0 +1,172 @@ +--- +title: Live Generation +id: live-generation +order: 6.5 +description: "Open a live, prompt-steerable video session with generateLiveVideo(). Mint a connect payload on the server, connect in the browser, and steer the stream with new prompts." +keywords: + - tanstack ai + - live generation + - generateLiveVideo + - reactor + - fal + - h3 max director + - helios + - live video + - experimental +--- + +# Live Generation (Experimental) + +You want a video that plays while it generates. You also want to change the prompt mid-run. A finite video job stops with a file. `generateLiveVideo()` opens a session instead. + +Call `generateLiveVideo()` on the server. It returns a connect payload: a model id, a prompt, and a token. The browser uses that payload to open the stream. + +> **Experimental.** The API can change. Live models bill per session-second while a GPU is held. + +## 1. Mint a session on the server + +Pick one adapter. They do not return a download URL. + +```ts group=live-reactor +import { generateLiveVideo } from '@tanstack/ai' +import { reactorVideo } from '@tanstack/ai-reactor' + +const live = await generateLiveVideo({ + adapter: reactorVideo('helios'), + prompt: 'A chef tosses noodles in a steel wok, flames leaping', +}) + +// Hand live.token, live.model, and live.prompt to the browser. +``` + +```ts group=live-fal +import { generateLiveVideo } from '@tanstack/ai' +import { falLiveVideo } from '@tanstack/ai-fal' + +const live = await generateLiveVideo({ + adapter: falLiveVideo('minimax/h3-max/director'), + prompt: 'Live shopping stream: a host holds up a gold watch to camera', +}) + +// live.model is 'fal-ai/minimax-h3-max-director'. Hand it to the browser. +``` + +Set `REACTOR_API_KEY` or `FAL_KEY`, or pass `apiKey` in the adapter config. Do not put the key in the JSON body. + +A full app lives in [`examples/ts-react-media`](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). + +## 2. Connect in the browser + +The server half is the same for every live adapter. The browser client is not. + +### Reactor + +Install `@reactor-team/js-sdk`. Connect with the token. Helios uses `set_sr_scale`, then `set_prompt`, then `start`. + +```ts group=live-reactor +import { Reactor } from '@reactor-team/js-sdk' + +const reactor = new Reactor({ modelName: live.model }) +const video = document.querySelector('video') + +reactor.on('trackReceived', (name, _track, stream) => { + if (name !== 'main_video') return + if (!video) return + video.muted = true + const attach = () => { + video.srcObject = null + video.srcObject = stream + void video.play().catch((error: unknown) => { + if (error instanceof DOMException && error.name === 'AbortError') return + }) + } + attach() + for (const track of stream.getTracks()) { + track.addEventListener('unmute', attach) + } +}) + +await reactor.connect(live.token) +await reactor.sendCommand('set_sr_scale', { sr_scale: '2x' }) +await reactor.sendCommand('set_prompt', { prompt: live.prompt }) +await reactor.sendCommand('start', {}) +``` + +FastH3 uses `enqueue` only. LongLive uses `set_shot` then `start`. + +Helios can also take a seed image. Pass a `File` from ``. Do not send base64. + +```ts group=live-reactor +const picker = document.querySelector('input[type="file"]') +if (!(picker instanceof HTMLInputElement)) { + throw new Error('Pick a seed image') +} +const file = picker.files?.[0] +if (file === undefined) { + throw new Error('Pick a seed image') +} +const image = await reactor.uploadFile(file) +await reactor.sendCommand('set_conditioning', { prompt: live.prompt, image }) +await reactor.sendCommand('start', {}) +``` + +A later `set_prompt` morphs the shot at the next chunk. + +See the [Reactor adapter](../adapters/reactor) for model ids. + +### fal H3 Max Director + +Install `@fal-ai/client@alpha`. Keep `FAL_KEY` on the server. `live.model` is the WMA app id. Open WMA through a proxy that attaches the key. Do not send `live.token` as `Key` credentials. + +The example proxy is `src/routes/api.fal.proxy.ts` in [`examples/ts-react-media`](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). It attaches the key and forwards only to `wma.fal.run` (`/ice`, `/session`, `/session/heartbeat`) and Director `/ice`. + +```ts group=live-fal +import { createFalClient } from '@fal-ai/client' +import { wma } from '@fal-ai/client/realtime' + +const fal = createFalClient({ proxyUrl: '/api/fal/proxy' }) +const video = document.querySelector('video') + +const session = fal.realtime.open(wma(live.model), { + receive: ['video', 'audio'], + onError: (error) => { + console.error(error) + }, + onMedia: (stream) => { + if (!video) return + video.muted = true + video.srcObject = stream + void video.play().catch((error: unknown) => { + if (error instanceof DOMException && error.name === 'AbortError') return + }) + }, +}) + +session.send({ + type: 'configure', + prompt: live.prompt, + prompt_version: 1, + protocol_version: 1, +}) + +await session.ready +``` + +To steer, send `{ type: 'prompt', prompt, prompt_version }` and increase `prompt_version` each time. To stop, send `{ type: 'stop' }` and close the session. + +Director bills a 60 second minimum. Resolution is `480p` or `768p`. See the [fal adapter](../adapters/fal). + +## Models + +| Adapter | Id | What it does | +| --- | --- | --- | +| `reactorVideo()` | `helios` | Interactive realtime video | +| `reactorVideo()` | `fast-h3` | Fast live clips on a live track | +| `reactorVideo()` | `longlive-v2` | Shot-based live clips | +| `falLiveVideo()` | `minimax/h3-max/director` | Steerable live stream over WMA | + +For Orbis and LingBot, use [World Generation](./world-generation). For a file that finishes, use [Video Generation](./video-generation). + +## What you have now + +A server call that returns a connect payload. The browser opens the stream and steers it until you stop. diff --git a/docs/media/video-generation.md b/docs/media/video-generation.md index f751d5d639..32409efec8 100644 --- a/docs/media/video-generation.md +++ b/docs/media/video-generation.md @@ -2,7 +2,7 @@ title: Video Generation id: video-generation order: 6 -description: "Generate video from text prompts with OpenAI Sora, Google Veo, Gemini Omni Flash, xAI Grok Imagine, BytePlus Seedance, OpenRouter, or fal.ai using TanStack AI's experimental generateVideo() jobs/polling API." +description: "Generate video from text prompts with OpenAI Sora, Google Veo, Gemini Omni Flash, xAI Grok Imagine, BytePlus Seedance, OpenRouter, or fal.ai using TanStack AI's experimental generateVideo() API." keywords: - tanstack ai - video generation @@ -39,12 +39,14 @@ keywords: ## Overview -TanStack AI provides experimental support for video generation through dedicated video adapters. Unlike image generation, video generation is an **asynchronous operation** that uses a jobs/polling pattern: +TanStack AI provides experimental support for video generation through dedicated video adapters. Most providers are **asynchronous** and use a jobs/polling pattern: 1. **Create a job** - Submit a prompt and receive a job ID 2. **Poll for status** - Check the job status until it's complete 3. **Retrieve the video** - Get the URL to download/view the generated video +For a prompt-steerable live stream (no download URL), use [Live Generation](./live-generation) or [World Generation](./world-generation). + Currently supported: - **OpenAI**: Sora-2 and Sora-2-Pro models (when available) @@ -80,6 +82,8 @@ const { jobId, model } = await generateVideo({ console.log("Job started:", jobId); ``` +For a stream that plays while it generates, and that you can steer with a new prompt, use [Live Generation](./live-generation). `generateVideo()` is the job path: create, poll, then fetch a file URL. + ### Polling for Status ```typescript @@ -925,9 +929,12 @@ await generateVideo({ #### VideoJobResult (from create) ```typescript +import type { PersistedArtifactRef } from '@tanstack/ai/client' + interface VideoJobResult { jobId: string; // Unique job identifier for polling model: string; // Model used for generation + artifacts?: Array } ``` diff --git a/docs/media/world-generation.md b/docs/media/world-generation.md new file mode 100644 index 0000000000..2b71697316 --- /dev/null +++ b/docs/media/world-generation.md @@ -0,0 +1,166 @@ +--- +title: World Generation +id: world-generation +order: 7 +description: "Open a live, prompt-steerable world session with generateWorld(). Mint a token on the server, connect in the browser, and steer the stream with new prompts." +keywords: + - tanstack ai + - world generation + - generateWorld + - reactor + - orbis + - infinite world + - live video + - experimental +--- + +# World Generation (Experimental) + +You want a world that generates while the viewer watches or changes the prompt. A finite video job stops. `generateWorld()` opens a live session instead. + +Call `generateWorld()` on the server. It returns a short-lived token, a model slug, and the prompt. The browser connects, sets the prompt, and starts the stream. LingBot also needs a seed image after connect. + +> **Experimental.** The API can change. World models bill per session-second while a GPU is held. + +## 1. Mint a session on the server + +The browser can paste a Reactor key. The relay reads `x-byok-reactor`, then `REACTOR_API_KEY`. Do not put the key in the JSON body. + +```ts +import { generateWorld } from '@tanstack/ai' +import { byokMissing, getByokKey } from '@tanstack/ai/byok/server' +import { reactorWorld } from '@tanstack/ai-reactor' +import { reactorByok } from '@tanstack/ai-reactor/byok' + +export async function POST(request: Request) { + const apiKey = getByokKey(request, reactorByok) + if (!apiKey) return byokMissing(reactorByok) + + const body = await request.json() + const prompt = typeof body.prompt === 'string' ? body.prompt : '' + if (prompt.length === 0) { + return Response.json({ error: 'prompt is required' }, { status: 400 }) + } + + const world = await generateWorld({ + adapter: reactorWorld('visko-orbis-stable', { apiKey }), + prompt, + }) + + return Response.json({ + token: world.token, + model: world.model, + prompt: world.prompt, + expiresAt: world.expiresAt, + }) +} +``` + +See [Bring Your Own Key](../advanced/byok) for the client store. + +## 2. Connect in the browser + +Install `@reactor-team/js-sdk`. Connect with the token. Then set the prompt and start. + +```ts group=world-browser +import { Reactor } from '@reactor-team/js-sdk' +import { defineByok, defaultByokStorage } from '@tanstack/ai-client/byok' +import { reactorByok } from '@tanstack/ai-reactor/byok' + +const byok = defineByok({ + storage: defaultByokStorage(), + providers: [reactorByok], +}) +byok.setServerCoverage(true) + +const video = document.querySelector('video') +if (!video) { + throw new Error('Missing video element') +} + +const world = await fetch('/api/world', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...byok.headers(reactorByok.id), + }, + body: JSON.stringify({ + prompt: + 'A dramatic coastline of black volcanic cliffs at golden hour, a single unbroken take.', + }), +}).then(async (response) => { + if (!response.ok) { + throw new Error('World session failed') + } + return response.json() +}) + +const token = typeof world.token === 'string' ? world.token : '' +const model = typeof world.model === 'string' ? world.model : '' +const prompt = typeof world.prompt === 'string' ? world.prompt : '' +if (token.length === 0 || model.length === 0 || prompt.length === 0) { + throw new Error('World payload is incomplete') +} + +const reactor = new Reactor({ modelName: model }) + +reactor.on('trackReceived', (name, _track, stream) => { + if (name !== 'main_video') return + video.muted = true + const attach = () => { + video.srcObject = null + video.srcObject = stream + void video.play().catch((error: unknown) => { + if (error instanceof DOMException && error.name === 'AbortError') return + }) + } + attach() + for (const track of stream.getTracks()) { + track.addEventListener('unmute', attach) + } +}) + +await reactor.connect(token) +await reactor.sendCommand('set_resolution', { resolution: '1080p' }) +await reactor.sendCommand('set_prompt', { prompt }) +await reactor.sendCommand('start', {}) +``` + +The video element now plays a live world. A new `set_prompt` during the run morphs the scene at the next chunk. + +LingBot and LingBot World 2 start from a seed image. Pass a `File` from ``. The SDK uploads the file. Do not send base64. `start` still needs `set_prompt`. Send a short default, then steer after the first frame. + +```ts group=world-browser +const picker = document.querySelector('input[type="file"]') +if (!(picker instanceof HTMLInputElement)) { + throw new Error('Pick a seed image') +} +const file = picker.files?.[0] +if (file === undefined) { + throw new Error('Pick a seed image') +} +const image = await reactor.uploadFile(file) +await reactor.sendCommand('set_image', { image }) +await reactor.sendCommand('set_prompt', { prompt: 'Follow the seed image.' }) +await reactor.sendCommand('start', {}) +``` + +## Models + +`reactorWorld()` accepts these ids. The result `model` field is the Reactor connect slug. + +| Id | Connect slug | What it does | +| --- | --- | --- | +| `visko-orbis-stable` | `reactor/visko-orbis-stable` | Steerable video with realtime audio | +| `visko-orbis-dynamic` | `reactor/visko-orbis-dynamic` | Same family, live resolution switch | +| `lingbot-world-2` | `reactor/lingbot-world-2` | Image-anchored navigable world | +| `lingbot` | `reactor/lingbot` | Image-anchored navigable video | +| `helios` | `reactor/helios` | Interactive realtime video | + +Happy Oyster (`happy-oyster-adventure`, `happy-oyster-director`) uses `createWorld` and `startTravel` after connect. See the [Reactor adapter](../adapters/reactor) for every id. Helios also works with `generateLiveVideo()` and `reactorVideo()`. See [Live Generation](./live-generation). + +A full app lives in [`examples/ts-react-media`](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). Open the World tab. + +## What you have now + +A server route that mints a world session, and a browser that streams it. Change the prompt while the video plays to steer the world. diff --git a/examples/README.md b/examples/README.md index 2e03103fdd..3241067124 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,6 +7,7 @@ This directory contains comprehensive examples demonstrating TanStack AI across Choose an example based on your use case: - **Want a full-stack TypeScript app?** → [TanStack Chat (ts-react-chat)](#tanstack-chat-ts-react-chat) +- **Need a live world stream?** → [World generation (ts-react-media)](#world-generation-ts-react-media) - **Need a vanilla JS frontend?** → [Vanilla Chat](#vanilla-chat) - **Multi-User TypeScript chat app?** → [Group Chat (ts-group-chat)](#group-chat-ts-group-chat) - **Polyglot AG-UI backends (Go/Rust/PHP/Zig/Bash/Python)?** → [AG-UI Polyglot Echo (ag-ui)](#ag-ui-polyglot-echo-ag-ui) @@ -128,6 +129,22 @@ pnpm dev --- +### World generation (ts-react-media) + +The media example has a **World** tab. Paste a Reactor key in the header +dialog. The relay calls `generateWorld()`. The page connects and can steer +the scene mid-run. + +```bash +cd examples/ts-react-media +pnpm install +pnpm dev +``` + +Open the World tab. Paste a Reactor key, or set `REACTOR_API_KEY`. + +--- + ### Vanilla Chat A framework-free chat application using pure JavaScript and `@tanstack/ai-client`. diff --git a/examples/ts-react-media/package.json b/examples/ts-react-media/package.json index 89f5d0b74f..7d054923a3 100644 --- a/examples/ts-react-media/package.json +++ b/examples/ts-react-media/package.json @@ -10,14 +10,18 @@ "test:types": "tsc" }, "dependencies": { + "@fal-ai/client": "1.11.0-alpha.2", + "@reactor-team/js-sdk": "^3.0.1", "@tailwindcss/vite": "^4.1.18", "@tanstack/ai": "workspace:*", "@tanstack/ai-byteplus": "workspace:*", + "@tanstack/ai-client": "workspace:*", "@tanstack/ai-fal": "workspace:*", "@tanstack/ai-gemini": "workspace:*", "@tanstack/ai-grok": "workspace:*", "@tanstack/ai-openrouter": "workspace:*", "@tanstack/ai-react": "workspace:*", + "@tanstack/ai-reactor": "workspace:*", "@tanstack/react-router": "^1.158.4", "@tanstack/react-start": "^1.159.0", "@tanstack/router-plugin": "^1.158.4", diff --git a/examples/ts-react-media/src/components/ByokKeyDialog.tsx b/examples/ts-react-media/src/components/ByokKeyDialog.tsx new file mode 100644 index 0000000000..a1d64c3b0c --- /dev/null +++ b/examples/ts-react-media/src/components/ByokKeyDialog.tsx @@ -0,0 +1,289 @@ +import { useEffect, useState } from 'react' +import { KeyRound, X } from 'lucide-react' +import { useByok } from '@tanstack/ai-react' +import { byok, KEYED_PROVIDERS } from '@/lib/byok' +import type { KeyStatus } from '@tanstack/ai-client/byok' +import type { ProviderId } from '@tanstack/ai/byok' + +export type ByokKeyDialogProps = { + open: boolean + onOpenChange: (open: boolean) => void + envStatus?: Record +} + +const STATUS_LABEL: Record< + KeyStatus['state'], + { label: string; className: string } +> = { + empty: { label: 'Not set', className: 'text-gray-500' }, + set: { label: 'Saved', className: 'text-gray-400' }, + locked: { label: 'Locked', className: 'text-amber-400' }, + error: { label: 'Check failed', className: 'text-red-400' }, +} + +export function ByokKeyDialog({ + open, + onOpenChange, + envStatus, +}: ByokKeyDialogProps) { + const snapshot = useByok(byok) + const [unlocking, setUnlocking] = useState(false) + const [actionError, setActionError] = useState('') + + useEffect(() => { + if (!open) return + const onKey = (event: KeyboardEvent) => { + if (event.key === 'Escape') onOpenChange(false) + } + window.addEventListener('keydown', onKey) + return () => window.removeEventListener('keydown', onKey) + }, [open, onOpenChange]) + + const locked = snapshot.locked + const needsAttention = + locked || + KEYED_PROVIDERS.every((provider) => { + const status = snapshot.status[provider.id] + const empty = !status || status.state === 'empty' + return empty && !envStatus?.[provider.id] + }) + + return ( + <> + + + {open ? ( +
onOpenChange(false)} + > +
+
event.stopPropagation()} + > +
+

API keys

+ +
+

+ Keys stay in this browser and are sent per request in a header. + Never stored on the server. Providers with a server env key + already work without one. +

+ + {snapshot.locked ? ( +
+ Saved keys are locked ({byok.storage.label}). + +
+ ) : null} + + {byok.storage.warning ? ( +

+ {byok.storage.warning} +

+ ) : null} + {snapshot.storageError ? ( +

+ {snapshot.storageError} +

+ ) : null} + {actionError ? ( +

{actionError}

+ ) : null} + +
+ {KEYED_PROVIDERS.map((provider) => ( + + ))} +
+
+
+
+ ) : null} + + ) +} + +function ProviderRow({ + id, + label, + status, + locked, + hasEnvKey, +}: { + id: ProviderId + label: string + status: KeyStatus | undefined + locked: boolean + hasEnvKey: boolean +}) { + const [draft, setDraft] = useState('') + const [rowError, setRowError] = useState('') + const state = status?.state ?? 'empty' + const masked = status && 'masked' in status ? status.masked : undefined + const isLocked = locked || state === 'locked' + const hasKey = state !== 'empty' + + return ( +
+
+ {label} + +
+ + {hasKey && masked ? ( +
+ + {masked} + + +
+ ) : null} + + {state === 'error' && status && 'message' in status ? ( +

{status.message}

+ ) : null} + {rowError ?

{rowError}

: null} + +
{ + event.preventDefault() + const next = draft.trim() + if (!next || isLocked) return + setRowError('') + void byok + .update(id, next) + .then(() => setDraft('')) + .catch((error: unknown) => + setRowError( + error instanceof Error ? error.message : 'Could not save key', + ), + ) + }} + > + setDraft(event.target.value)} + className="flex-1 rounded-md border border-gray-600 bg-gray-950 px-2 py-1.5 text-sm text-white placeholder-gray-500 disabled:opacity-50" + /> + +
+
+ ) +} + +function PresenceBadge({ + state, + hasKey, + isLocked, + hasEnvKey, +}: { + state: KeyStatus['state'] + hasKey: boolean + isLocked: boolean + hasEnvKey: boolean +}) { + if (hasKey && !isLocked) { + return ( + Your key + ) + } + if (isLocked) { + return Locked + } + if (hasEnvKey) { + return ( + Server key + ) + } + const { label, className } = STATUS_LABEL[state] + return {label} +} diff --git a/examples/ts-react-media/src/components/Header.tsx b/examples/ts-react-media/src/components/Header.tsx index ee8fa53c87..687e5b93ab 100644 --- a/examples/ts-react-media/src/components/Header.tsx +++ b/examples/ts-react-media/src/components/Header.tsx @@ -1,10 +1,32 @@ +import { useEffect, useState } from 'react' import { Link } from '@tanstack/react-router' +import { useByok } from '@tanstack/ai-react' +import { ByokKeyDialog } from '@/components/ByokKeyDialog' +import { byok, getEnvKeyStatus } from '@/lib/byok' const navLinkClass = 'px-3 py-1.5 rounded-lg text-sm font-medium text-gray-300 hover:bg-gray-700 hover:text-white transition-colors' const navLinkActiveClass = 'bg-gray-700 text-white' export default function Header() { + const [keyDialogOpen, setKeyDialogOpen] = useState(false) + const [envStatus, setEnvStatus] = useState>({}) + const snapshot = useByok(byok) + + useEffect(() => { + void byok.ready() + }, []) + + useEffect(() => { + void getEnvKeyStatus() + .then(setEnvStatus) + .catch(() => setEnvStatus({})) + }, []) + + useEffect(() => { + if (snapshot.prompt) setKeyDialogOpen(true) + }, [snapshot.prompt]) + return (

@@ -16,7 +38,7 @@ export default function Header() { Image & Video Generation -

) diff --git a/examples/ts-react-media/src/components/ImageGenerator.tsx b/examples/ts-react-media/src/components/ImageGenerator.tsx index af18858fbc..921d2adb6d 100644 --- a/examples/ts-react-media/src/components/ImageGenerator.tsx +++ b/examples/ts-react-media/src/components/ImageGenerator.tsx @@ -4,6 +4,7 @@ import { useGenerateImage } from '@tanstack/ai-react' import type { MediaPrompt } from '@tanstack/ai/client' import { generateImageFn } from '@/lib/server-functions' +import { byok, callWithByok, toByokProvider } from '@/lib/byok' import { getRandomImagePrompt } from '@/lib/prompts' import { IMAGE_MODELS } from '@/lib/models' import type { ImageModel } from '@/lib/models' @@ -292,11 +293,16 @@ function ImageModelCard({ // switch is picked here rather than being sent as a request field. // `options.signal` is the hook's abort signal — forwarding it lets an // unmount or a `stop()` cancel the request rather than orphan it. + byok, + byokProvider: () => toByokProvider(model.provider), fetcher: (input, options) => - generateImageFn({ - data: { prompt: input.prompt, model: model.id }, - signal: options?.signal, - }), + callWithByok( + generateImageFn({ + data: { prompt: input.prompt, model: model.id }, + signal: options?.signal, + headers: options?.headers, + }), + ), onResult: (generated) => { const image = generated.images[0] if (image) onImageGenerated?.(getImageSrc(image)) diff --git a/examples/ts-react-media/src/components/LiveVideoStudio.tsx b/examples/ts-react-media/src/components/LiveVideoStudio.tsx new file mode 100644 index 0000000000..fc4aa164de --- /dev/null +++ b/examples/ts-react-media/src/components/LiveVideoStudio.tsx @@ -0,0 +1,564 @@ +import { useEffect, useRef, useState } from 'react' +import { Loader2, Square, TriangleAlert } from 'lucide-react' +import { generateLiveVideoFn } from '@/lib/server-functions' +import { attachStream } from '@/lib/attach-stream' +import { SeedImageField } from '@/components/SeedImageField' +import { + byok, + callWithByok, + falByok, + reactorByok, + requestByokFromError, +} from '@/lib/byok' +import { + FAL_LIVE_VIDEO_APP, + LIVE_VIDEO_MODEL_LABELS, + LIVE_VIDEO_MODELS, + LIVE_VIDEO_PROMPTS, + isFalLiveVideoModel, + isLiveVideoModelId, + liveVideoProvider, + liveVideoResolutions, +} from '@/lib/models' +import { + liveAcceptsSeedImage, + watchReactorFailure, +} from '@/lib/reactor-session' +import type { Reactor } from '@reactor-team/js-sdk' +import type { + LiveVideoModelId, + LiveVideoProvider, + LiveVideoResolution, +} from '@/lib/models' + +type SessionStatus = 'idle' | 'connecting' | 'live' | 'error' + +type FalLiveSession = { + send: (message: Record) => void + close: () => void | Promise +} + +type LiveHandle = + | { provider: 'reactor'; reactor: Reactor; model: LiveVideoModelId } + | { provider: 'fal'; session: FalLiveSession } + +async function startReactorLive( + reactor: Reactor, + model: LiveVideoModelId, + prompt: string, + resolution: LiveVideoResolution, + seedFile: File | null, +): Promise { + if (model === 'fast-h3') { + await reactor.sendCommand('enqueue', { prompt }) + return + } + if (model === 'longlive-v2') { + await reactor.sendCommand('set_shot', { prompt }) + await reactor.sendCommand('start', {}) + return + } + if (model === 'helios') { + await reactor.sendCommand('set_sr_scale', { + sr_scale: resolution === '2k' || resolution === '4k' ? '4x' : '2x', + }) + if (seedFile) { + const image = await reactor.uploadFile(seedFile) + await reactor.sendCommand('set_conditioning', { prompt, image }) + } else { + await reactor.sendCommand('set_prompt', { prompt }) + } + await reactor.sendCommand('start', {}) + return + } + if (resolution === '1080p' || resolution === '2k' || resolution === '4k') { + await reactor.sendCommand('set_resolution', { resolution }) + } + await reactor.sendCommand('set_prompt', { prompt }) + await reactor.sendCommand('start', {}) +} + +async function steerReactorLive( + reactor: Reactor, + model: LiveVideoModelId, + prompt: string, +): Promise { + if (model === 'fast-h3') { + await reactor.sendCommand('enqueue', { prompt }) + return + } + if (model === 'longlive-v2') { + await reactor.sendCommand('set_shot', { prompt }) + return + } + await reactor.sendCommand('set_prompt', { prompt }) +} + +function errorMessage(error: unknown): string { + if (!(error instanceof Error)) return String(error) + const cause = error.cause + if (cause instanceof Error && cause.message.length > 0) { + return `${error.message} ${cause.message}` + } + return error.message +} + +function readLivePayload(value: unknown): { + token: string + model: string + prompt: string + provider: LiveVideoProvider +} { + if (typeof value !== 'object' || value === null) { + throw new Error('Live video payload is incomplete') + } + const token = + 'token' in value && typeof value.token === 'string' ? value.token : '' + const model = + 'model' in value && typeof value.model === 'string' ? value.model : '' + const prompt = + 'prompt' in value && typeof value.prompt === 'string' ? value.prompt : '' + const provider = + 'provider' in value && + (value.provider === 'reactor' || value.provider === 'fal') + ? value.provider + : null + if ( + token.length === 0 || + model.length === 0 || + prompt.length === 0 || + provider === null + ) { + throw new Error('Live video payload is incomplete') + } + return { token, model, prompt, provider } +} + +function defaultResolution(provider: LiveVideoProvider): LiveVideoResolution { + return provider === 'fal' ? '768p' : '1080p' +} + +async function openFalSession(args: { + model: string + prompt: string + resolution: LiveVideoResolution + onMedia: (stream: MediaStream) => void + onError: (error: unknown) => void +}): Promise { + const { createFalClient } = await import('@fal-ai/client') + const { wma } = await import('@fal-ai/client/realtime') + const fal = createFalClient({ + proxyUrl: '/api/fal/proxy', + fetch: (input, init) => { + const headers = new Headers(init?.headers) + for (const [name, value] of Object.entries(byok.headers(falByok.id))) { + headers.set(name, value) + } + return fetch(input, { ...init, headers }) + }, + }) + const wmaApp = isFalLiveVideoModel(args.model) + ? FAL_LIVE_VIDEO_APP[args.model] + : args.model + const session = fal.realtime.open(wma(wmaApp), { + receive: ['video', 'audio'], + onMedia: args.onMedia, + onError: args.onError, + onData: (raw) => { + let parsed: unknown + try { + parsed = JSON.parse(String(raw)) + } catch { + return + } + if ( + typeof parsed === 'object' && + parsed !== null && + 'type' in parsed && + parsed.type === 'error' && + 'error' in parsed && + typeof parsed.error === 'string' + ) { + args.onError(new Error(parsed.error)) + } + }, + onState: (state) => { + if (state === 'failed') args.onError(new Error('Live session failed')) + }, + }) + session.send({ + type: 'configure', + prompt: args.prompt, + prompt_version: 1, + protocol_version: 1, + ...(args.resolution === '480p' || args.resolution === '768p' + ? { resolution: args.resolution } + : {}), + }) + await session.ready + return session +} + +export default function LiveVideoStudio() { + const [prompt, setPrompt] = useState(LIVE_VIDEO_PROMPTS[0] ?? '') + const [steerPrompt, setSteerPrompt] = useState('') + const [model, setModel] = useState('helios') + const [resolution, setResolution] = useState('1080p') + const [seedFile, setSeedFile] = useState(null) + const [status, setStatus] = useState('idle') + const [error, setError] = useState(null) + const [playing, setPlaying] = useState(false) + const videoRef = useRef(null) + const handleRef = useRef(null) + const detachStreamRef = useRef<(() => void) | null>(null) + const unwatchRef = useRef<(() => void) | null>(null) + const falPromptVersionRef = useRef(1) + const statusRef = useRef('idle') + statusRef.current = status + + const provider = liveVideoProvider(model) + const resolutions = liveVideoResolutions(provider) + const keyProvider = provider === 'fal' ? falByok : reactorByok + const showSeedField = liveAcceptsSeedImage(model) + + async function teardown() { + unwatchRef.current?.() + unwatchRef.current = null + const handle = handleRef.current + handleRef.current = null + detachStreamRef.current?.() + detachStreamRef.current = null + if (handle?.provider === 'reactor') { + try { + await handle.reactor.disconnect() + } catch { + // Disconnect can fail if the session already ended. + } + } else if (handle?.provider === 'fal') { + try { + handle.session.send({ type: 'stop' }) + await handle.session.close() + } catch { + // Close can fail if the session already ended. + } + } + const video = videoRef.current + if (video) { + video.srcObject = null + } + } + + async function stop() { + await teardown() + setPlaying(false) + setStatus('idle') + } + + useEffect(() => { + return () => { + void teardown() + } + }, []) + + async function start() { + setError(null) + setPlaying(false) + setStatus('connecting') + try { + await byok.prepare(keyProvider.id) + const live = readLivePayload( + await callWithByok( + generateLiveVideoFn({ + data: { prompt, model, resolution }, + headers: byok.headers(keyProvider.id), + }), + ), + ) + const video = videoRef.current + if (!video) throw new Error('Video element is missing') + + if (live.provider === 'fal') { + falPromptVersionRef.current = 1 + const session = await openFalSession({ + model: live.model, + prompt: live.prompt, + resolution, + onMedia: (stream) => { + detachStreamRef.current?.() + detachStreamRef.current = attachStream( + video, + stream, + (playError) => { + setError(errorMessage(playError)) + }, + ) + }, + onError: (openError) => { + setError(errorMessage(openError)) + setStatus('error') + }, + }) + handleRef.current = { provider: 'fal', session } + } else { + const { Reactor: ReactorClient } = await import('@reactor-team/js-sdk') + const reactor = new ReactorClient({ + modelName: live.model, + }) + handleRef.current = { provider: 'reactor', reactor, model } + + let rejectStart: ((error: Error) => void) | undefined + const startFailed = new Promise((_, reject) => { + rejectStart = reject + }) + unwatchRef.current = watchReactorFailure(reactor, (message) => { + setError(message) + rejectStart?.(new Error(message)) + if (statusRef.current === 'live') setStatus('error') + }) + reactor.on('trackReceived', (name, _track, stream) => { + if (name !== 'main_video') return + detachStreamRef.current?.() + detachStreamRef.current = attachStream(video, stream, (playError) => { + setError(errorMessage(playError)) + }) + }) + + await reactor.connect(live.token) + try { + await Promise.race([ + startReactorLive(reactor, model, live.prompt, resolution, seedFile), + startFailed, + ]) + } finally { + rejectStart = undefined + } + } + + setSteerPrompt('') + setStatus('live') + } catch (caught) { + await stop() + requestByokFromError(caught) + setError(errorMessage(caught)) + setStatus('error') + } + } + + async function steer() { + const handle = handleRef.current + const next = steerPrompt.trim() + if (!handle || next.length === 0) return + setError(null) + try { + if (handle.provider === 'fal') { + falPromptVersionRef.current += 1 + handle.session.send({ + type: 'prompt', + prompt: next, + prompt_version: falPromptVersionRef.current, + }) + } else { + await steerReactorLive(handle.reactor, handle.model, next) + } + setPrompt(next) + setSteerPrompt('') + } catch (caught) { + setError(errorMessage(caught)) + } + } + + const isLive = status === 'live' + const isBusy = status === 'connecting' + + return ( +
+

+ Live video. Paste a key in the header dialog, or set{' '} + + {provider === 'fal' ? 'FAL_KEY' : 'REACTOR_API_KEY'} + {' '} + on the server. Then start a session and type under the view to steer. +

+ +
+
+ +
{ + event.preventDefault() + if (isLive) { + void steer() + return + } + if (!isBusy && prompt.trim().length > 0) void start() + }} + > +