diff --git a/.changeset/native-files-api-support.md b/.changeset/native-files-api-support.md new file mode 100644 index 0000000000..ce4096e919 --- /dev/null +++ b/.changeset/native-files-api-support.md @@ -0,0 +1,25 @@ +--- +'@tanstack/ai': minor +'@tanstack/ai-event-client': minor +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +'@tanstack/ai-anthropic': minor +'@tanstack/ai-gemini': minor +'@tanstack/ai-fal': minor +'@tanstack/ai-mistral': patch +'@tanstack/ai-grok': minor +'@tanstack/ai-openrouter': patch +'@tanstack/ai-ollama': patch +'@tanstack/ai-bedrock': patch +'@tanstack/ai-byteplus': patch +'@tanstack/ai-cohere': patch +--- + +feat(ai): native Files API support across providers (upload adapters + `file` content source) + +Adds first-class support for provider **Files / storage APIs** so callers can upload media once and reference it by a provider-issued handle instead of re-sending base64 or a public URL each request (lower latency/bandwidth, no re-buffering on memory-constrained runtimes). + +- **New tree-shakeable `files` adapter kind** — `openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, `grokFiles()`, and `falFiles()`. Each exposes `upload()`, and (where the provider has a lifecycle API) `get()` / `delete()`. Drive them with the new `uploadFile()` / `getFile()` / `deleteFile()` activity functions. fal is upload-only. +- **New `{ type: 'file' }` arm on `ContentPartSource`** — a **per-provider reference record**: `{ type: 'file', reference: { openai: 'file-…', gemini: 'https://…' } }`. Each adapter reads only its own entry and maps it to its native wire field: OpenAI (Responses) `input_image`/`input_file` `file_id`, Anthropic `file_id` message source (with the `files-api-2025-04-14` beta), Gemini `fileData.fileUri`, fal storage URL passthrough. `fileSourceFromHandle(...handles)` builds the source and merges handles from several providers into one source that routes to any of them. +- **Fail-closed capability preflight** — adapters that can consume file references declare `supportsFileSources`; `chat()` / `generateImage()` / `generateVideo()` / `embed()` reject `{ type: 'file' }` sources for every other adapter (Bedrock, Mistral, Groq, OpenRouter, Ollama, BytePlus, Cohere, and any future adapter that doesn't opt in) **before a request is built**, so a reference can never be silently mis-mapped onto a URL/data field. Endpoints that need raw bytes (image edits, Sora `input_reference`, Veo, Chat Completions images) throw endpoint-specific errors. A supporting adapter with no entry for its provider in the record throws a lookup error naming the providers that are present. +- **Provider-literal typed handles** — `FileHandle<'openai'>` etc. flow from each files adapter through `uploadFile()`, and `getFile()`/`deleteFile()` accept the handle itself, so cross-provider lifecycle calls fail at compile time. `fileSourceFromHandle` and `FileHandle` are also exported from the browser-safe `@tanstack/ai/client` entry. A `{ type: 'file' }` source cannot cross the chat wire format (which carries `data`/`url` sources only) and throws rather than being dropped, so a browser that holds a handle sends it in its own request body and the server builds the source. diff --git a/docs/advanced/files-api.md b/docs/advanced/files-api.md new file mode 100644 index 0000000000..bfd384aebf --- /dev/null +++ b/docs/advanced/files-api.md @@ -0,0 +1,245 @@ +--- +title: Files API +id: files-api +description: "Upload media once and reference it by a provider-issued handle with TanStack AI's tree-shakeable files adapters (OpenAI, Anthropic, Gemini, Grok, fal)." +keywords: + - tanstack ai + - files api + - file upload + - file_id + - fileData + - public url + - multimodal +--- + +Provider **Files / storage APIs** let you upload a media asset once and reference it later by a lightweight handle, instead of re-sending base64 (or relying on the provider to re-fetch a public URL) on every request. That means large or reused inputs are uploaded a single time — lower latency and bandwidth, no re-buffering of base64 on memory-constrained runtimes (e.g. Cloudflare Workers) — plus access to provider-side file lifecycle (TTL, deletion). + +TanStack AI exposes this as a tree-shakeable **`files` adapter** per provider, paired with a `{ type: 'file' }` [content source](./multimodal-content.md#file-handle-files-api) you drop into a message. + +## Files adapters + +Each provider with a native surface has a factory: `openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, `grokFiles()`, and `falFiles()`. They read the same API-key env var as the provider's other adapters. To pass a key explicitly, use the `create*Files(apiKey)` variants (`createOpenaiFiles`, `createAnthropicFiles`, `createGeminiFiles`, `createGrokFiles`). `falFiles(config)` takes its key in the config object. + +```typescript +import { createOpenaiFiles, openaiFiles } from '@tanstack/ai-openai' +import { geminiFiles } from '@tanstack/ai-gemini' +import { anthropicFiles } from '@tanstack/ai-anthropic' +import { falFiles } from '@tanstack/ai-fal' +import { grokFiles } from '@tanstack/ai-grok' + +const files = openaiFiles() // reads OPENAI_API_KEY +const filesWithKey = createOpenaiFiles('sk-your-key') // explicit key +``` + +### uploadFile + +Drive an adapter with the `uploadFile()` activity function. It accepts a `Blob` (memory-efficient — preferred for large assets) or `{ data, mimeType }` where `data` is base64, and returns a `FileHandle`: + +```typescript +import { uploadFile } from '@tanstack/ai' +import { openaiFiles } from '@tanstack/ai-openai' +import { pdfBase64 } from './pdf-data' + +const handle = await uploadFile({ + adapter: openaiFiles(), + input: { data: pdfBase64, mimeType: 'application/pdf' }, +}) +// handle: { id, provider, uri?, mimeType?, sizeBytes?, expiresAt?, filename? } +``` + +- `id` — the provider handle used for `get` / `delete` (OpenAI/Anthropic `file_id`, Gemini file resource name, fal storage URL). +- `uri`: the handle's URL form when the provider exposes one (Gemini file URI, fal storage URL, Grok public URL). It is `undefined` for OpenAI and Anthropic, whose handles are opaque ids. +- `expiresAt` — epoch milliseconds, when the provider schedules the handle to expire. + +> **Runtime note (Gemini upload).** `geminiFiles().upload()` uses `@google/genai`'s +> resumable upload, which sets an explicit `Content-Length` header on a `Blob`-body +> request. Some server runtimes reject that with `fetch failed` / +> `InvalidArgumentError: invalid content-length header`. On **TanStack Start / Nitro** +> this fails on older Nitro (observed on `nitro@3.0.1-alpha.2`) and works on current +> Nitro (verified on `nitro@3.0.260610-beta`) — upgrade Nitro if you hit it. Native +> Node (and the production `node-server` build) are unaffected. OpenAI, Anthropic, and +> fal uploads use different transports and don't exercise this path. + +### getFile and deleteFile + +Providers with a lifecycle API support `getFile()` and `deleteFile()`. Both accept the handle itself (preferred — the handle's provider type rejects a foreign provider's handle at compile time) or its raw `id`: + +```typescript +import { deleteFile, getFile, uploadFile } from '@tanstack/ai' +import { openaiFiles } from '@tanstack/ai-openai' +import { pdfBase64 } from './pdf-data' + +const files = openaiFiles() +const handle = await uploadFile({ + adapter: files, + input: { data: pdfBase64, mimeType: 'application/pdf' }, +}) + +const meta = await getFile({ adapter: files, id: handle }) +await deleteFile({ adapter: files, id: handle }) +``` + +> fal storage is **upload-only** — `falFiles()` defines no `get` / `delete`, and calling `getFile()` / `deleteFile()` with it throws a clear error. + +### Grok (xAI): handles are public URLs + +`grokFiles()` uploads to the xAI Files API, then mints a [public URL](https://docs.x.ai/developers/files/public-urls) for the stored object and uses that URL as the handle's reference. + +xAI takes a `file_id` only on `input_file` (documents), and only on agentic-capable models. Its image path takes a URL. A public URL works for both, so one handle covers every modality on every chat model. + +```typescript +import { uploadFile } from '@tanstack/ai' +import { grokFiles } from '@tanstack/ai-grok' +import { pngBase64 } from './image-data' + +const handle = await uploadFile({ + // Omit `expiresAfter` for a URL that does not expire. + adapter: grokFiles({ expiresAfter: 86_400 }), + input: { data: pngBase64, mimeType: 'image/png' }, +}) +// handle.id -> 'file_abc123' (lifecycle) +// handle.uri -> 'https://files-cdn.x.ai/…' (wire reference) +``` + +xAI limits to know: + +- 50 MiB per file, and PNG, JPEG, MP4, or PDF only. +- `expiresAfter` runs from 3600 seconds (one hour) to 2592000 (thirty days). +- Up to 1000 active public URLs per team. +- Minting is idempotent, so re-uploading the same file returns the same URL. + +To stop a URL resolving without deleting the file, call `revokePublicUrl(handle.id)` on the adapter. `deleteFile()` removes the file itself. + +## Referencing a handle in a message + +Use `fileSourceFromHandle(handle)` to turn a `FileHandle` into a `{ type: 'file' }` content source. The source carries a **record of per-provider references** (`{ reference: { openai: 'file-abc' } }`). Each adapter reads only its own entry and maps it to its native wire field: OpenAI and Anthropic `file_id`, Gemini `fileData.fileUri`, fal storage URL, Grok public URL. Sending the source to a provider with no entry in the record throws a clear error, and adapters that can't consume file references at all are rejected before any mapping starts. + +### Server: upload + reference + +```typescript +import { chat, fileSourceFromHandle, uploadFile } from '@tanstack/ai' +import { anthropicFiles, anthropicText } from '@tanstack/ai-anthropic' + +export async function askAboutPdf(pdfBase64: string, request: string) { + // Upload once; reuse the handle across turns. + const handle = await uploadFile({ + adapter: anthropicFiles(), + input: { data: pdfBase64, mimeType: 'application/pdf' }, + }) + + return chat({ + adapter: anthropicText('claude-sonnet-5'), + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: request }, + { type: 'document', source: fileSourceFromHandle(handle) }, + ], + }, + ], + }) +} +``` + +### One source, several providers + +Because `reference` is a record, the same bytes uploaded to two providers merge into **one** source that routes correctly to either — useful when a conversation may be replayed against different models: + +```typescript +import { chat, fileSourceFromHandle, uploadFile } from '@tanstack/ai' +import { openaiFiles, openaiText } from '@tanstack/ai-openai' +import { geminiFiles } from '@tanstack/ai-gemini' +import { pdfBase64 } from './pdf-data' + +const input = { data: pdfBase64, mimeType: 'application/pdf' } +const openaiHandle = await uploadFile({ adapter: openaiFiles(), input }) +const geminiHandle = await uploadFile({ adapter: geminiFiles(), input }) + +// reference: { openai: 'file-…', gemini: 'https://…/files/…' } +const source = fileSourceFromHandle(openaiHandle, geminiHandle) + +chat({ + adapter: openaiText('gpt-5.5'), // or a gemini adapter — same message works + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this document' }, + { type: 'document', source }, + ], + }, + ], +}) +``` + +### Client: reuse a handle across requests + +Upload needs the provider key, so it happens on the server. The browser holds the handle it gets back and sends that handle with each turn. + +Send the handle in your own request body, not in the message content. The chat wire format carries `data` and `url` sources only, so a `{ type: 'file' }` source cannot cross it. Build the source on the server instead. + +1. Store the handle the upload endpoint returned. Keep `{ id, provider, uri, mimeType }`. +2. Put that handle in the request body, next to the messages. +3. On the server, call `fileSourceFromHandle` and add the part to the message. + +In the browser, pass the handle through the `body` option: + +```tsx +import { useChat } from '@tanstack/ai-react' +import { fetchServerSentEvents } from '@tanstack/ai-client' +import type { FileHandle } from '@tanstack/ai/client' + +// `handle` came from your upload endpoint and is stored client-side. +function AskAboutFile({ handle }: { handle: FileHandle }) { + const { sendMessage } = useChat({ + connection: fetchServerSentEvents('/api/chat'), + }) + + return ( + + ) +} +``` + +On the server, attach the file part before the run starts: + +```typescript +import { chat, fileSourceFromHandle } from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import type { FileHandle, ModelMessage } from '@tanstack/ai' + +export function runTurn(messages: Array, handle: FileHandle) { + const last = messages[messages.length - 1] + if (last?.role === 'user' && Array.isArray(last.content)) { + last.content.push({ type: 'image', source: fileSourceFromHandle(handle) }) + } + + return chat({ adapter: openaiText('gpt-5.5'), messages }) +} +``` + +> A `file` source sent through the chat wire throws with a clear error. It is never dropped and never sent as a URL. Support for handles in the wire format is tracked in [ag-ui#2639](https://github.com/ag-ui-protocol/ag-ui/issues/2639). + +## Provider support + +| Provider | Adapter | Handle referenced as | Lifecycle | +| --- | --- | --- | --- | +| OpenAI | `openaiFiles()` | Responses `input_image` / `input_file` `file_id` | `get`, `delete` | +| Anthropic | `anthropicFiles()` | `file_id` message source (sends the `files-api-2025-04-14` beta) | `get`, `delete` | +| Gemini | `geminiFiles()` | `fileData.fileUri` (the handle URI) | `get`, `delete` | +| fal | `falFiles()` | storage URL (used like any URL) | upload-only | +| Grok (xAI) | `grokFiles()` | public URL (used like any URL) | `get`, `delete` | + +Gemini and fal handles are URLs, so they also round-trip through a plain `{ type: 'url' }` source; OpenAI and Anthropic handles are opaque ids that require the `{ type: 'file' }` source. + +### Providers and endpoints that can't consume references + +Adapters that can consume file references declare a `supportsFileSources` capability; for everyone else (Groq, Bedrock, Mistral, OpenRouter, Ollama, BytePlus, Cohere, and any adapter written before this feature existed) `chat()` / `generateImage()` / `generateVideo()` / `embed()` reject `{ type: 'file' }` sources **before any request is built**, so a reference can never be silently mis-mapped onto a URL or data field. + +Some endpoints on supporting providers also have no "reference an uploaded handle" option — OpenAI's `images/edits` and Sora `input_reference`, and Gemini's Veo, need the actual bytes (or, for Veo, a `gs://` URI). The OpenAI **Chat Completions** image path also references images only by URL/data URI, not `file_id` — use the Responses adapter (`openaiText`) for `file_id` images. These throw a clear endpoint-specific error. diff --git a/docs/advanced/multimodal-content.md b/docs/advanced/multimodal-content.md index baa86b534f..314a2ed533 100644 --- a/docs/advanced/multimodal-content.md +++ b/docs/advanced/multimodal-content.md @@ -279,6 +279,40 @@ const imagePart = { **Note:** Not all providers support URL-based content for all modalities. Check provider documentation for specifics. +### File Handle (Files API) + +Use `type: 'file'` to reference media you uploaded once via a provider's [Files API](./files-api.md) — the provider stores the bytes and you pass a lightweight reference instead of re-sending base64 or a public URL every request. The source carries a record of per-provider references (`{ reference: { openai: 'file-…' } }`); each adapter reads its own entry and throws when none is present, and adapters without Files API support reject the source before any request is built. + +```typescript +import { openaiFiles, openaiText } from '@tanstack/ai-openai' +import { chat, fileSourceFromHandle, uploadFile } from '@tanstack/ai' +import { pdfBase64 } from './pdf-data' + +// Upload once... +const handle = await uploadFile({ + adapter: openaiFiles(), + input: { data: pdfBase64, mimeType: 'application/pdf' }, +}) + +// ...then reference the handle by id in as many requests as you like. +for await (const chunk of chat({ + adapter: openaiText('gpt-5.5'), + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this document' }, + { type: 'document', source: fileSourceFromHandle(handle) }, + ], + }, + ], +})) { + // ... +} +``` + +`fileSourceFromHandle(...handles)` builds the `{ type: 'file', reference }` source for you (picking the handle URL for Gemini/fal or the opaque id for OpenAI/Anthropic), and merges handles from several providers into one source that routes to any of them. Each adapter maps its own reference entry to the provider's native field (`file_id`, `fileData.fileUri`, or storage URL). Sending the source to a provider with no entry — or to an endpoint that requires raw bytes (image edits, Veo) — throws a clear error. See [Files API](./files-api.md) for uploading, retrieving, and deleting handles. + ## Backward Compatibility String content continues to work as before: diff --git a/docs/config.json b/docs/config.json index 5439a2d1d2..ccd7d0aa91 100644 --- a/docs/config.json +++ b/docs/config.json @@ -908,7 +908,13 @@ "label": "Multimodal Content", "to": "advanced/multimodal-content", "addedAt": "2026-04-15", - "updatedAt": "2026-07-21" + "updatedAt": "2026-08-07" + }, + { + "label": "Files API", + "to": "advanced/files-api", + "addedAt": "2026-09-11", + "updatedAt": "2026-09-12" }, { "label": "Per-Model Type Safety", diff --git a/examples/ts-react-chat/src/routes/index.tsx b/examples/ts-react-chat/src/routes/index.tsx index 9fdca10439..dbf02113f9 100644 --- a/examples/ts-react-chat/src/routes/index.tsx +++ b/examples/ts-react-chat/src/routes/index.tsx @@ -115,7 +115,8 @@ function Messages({ const hasRenderablePart = (message: UIMessage): boolean => { return message.parts.some((part) => { if (part.type === 'thinking') return true - if (part.type === 'image') return true + // File-handle images have no local bytes or URL to show. + if (part.type === 'image' && 'value' in part.source) return true if (part.type === 'text' && part.content.trim()) return true if ( part.type === 'tool-call' && @@ -372,8 +373,9 @@ function Messages({ ) } - // Render image parts - if (part.type === 'image') { + // Render image parts (file references have no local bytes + // or URL to render, so only url/data sources get an ) + if (part.type === 'image' && 'value' in part.source) { const imageUrl = part.source.type === 'url' ? part.source.value diff --git a/examples/ts-react-media/src/components/ImageGenerator.tsx b/examples/ts-react-media/src/components/ImageGenerator.tsx index 921d2adb6d..26333213f8 100644 --- a/examples/ts-react-media/src/components/ImageGenerator.tsx +++ b/examples/ts-react-media/src/components/ImageGenerator.tsx @@ -183,7 +183,8 @@ export default function ImageGenerator({ Sent as image prompt parts with role "reference" — - accepted by the Gemini multimodal models, xAI Imagine and Seedream + accepted by the Gemini multimodal models (uploaded once via the + Gemini Files API), xAI Imagine and Seedream
diff --git a/examples/ts-react-media/src/lib/server-functions.ts b/examples/ts-react-media/src/lib/server-functions.ts index 340f5ce154..8a1d939499 100644 --- a/examples/ts-react-media/src/lib/server-functions.ts +++ b/examples/ts-react-media/src/lib/server-functions.ts @@ -1,12 +1,17 @@ import { createServerFn } from '@tanstack/react-start' import { getRequest } from '@tanstack/react-start/server' import { + falFiles, falImage, falLiveVideo, falVideo, isFalLiveVideoModel, } from '@tanstack/ai-fal' -import { createGeminiImage, createGeminiVideo } from '@tanstack/ai-gemini' +import { + createGeminiFiles, + createGeminiImage, + createGeminiVideo, +} from '@tanstack/ai-gemini' import { createGrokImage, createGrokVideo } from '@tanstack/ai-grok' import { createOpenRouterVideo } from '@tanstack/ai-openrouter' import { @@ -19,11 +24,13 @@ import { supportsReferenceMedia, } from '@tanstack/ai-byteplus' import { + fileSourceFromHandle, generateImage, generateLiveVideo, generateVideo, generateWorld, toServerSentEventsResponse, + uploadFile, } from '@tanstack/ai' import { byokMissing, getByokKey } from '@tanstack/ai/byok/server' import { @@ -52,7 +59,7 @@ import { WORLD_RESOLUTIONS, } from './models' -import type { StreamChunk } from '@tanstack/ai' +import type { FilesAdapter, StreamChunk } from '@tanstack/ai' import type { BytePlusVideoModel, BytePlusVideoModelOrString, @@ -145,6 +152,32 @@ function asImageToVideoPrompt( return narrowed } +/** + * Upload each inline (base64 `data`) image input to the provider's Files API and + * swap in a `{ type: 'file' }` handle. A reference image / start frame is then + * uploaded once via the tree-shakeable files adapter (`geminiF()` / + * `falF()`, the BYOK-keyed wrappers below) instead of being re-sent inline + * as base64 on the generation + * request — the memory-safe path for large inputs. URL and already-uploaded + * sources pass through untouched. + */ +async function uploadInlineImageInputs( + prompt: string | Array>, + files: FilesAdapter, +): Promise>> { + if (typeof prompt === 'string') return prompt + return Promise.all( + prompt.map(async (part) => { + if (part.type !== 'image' || part.source.type !== 'data') return part + const handle = await uploadFile({ + adapter: files, + input: { data: part.source.value, mimeType: part.source.mimeType }, + }) + return { ...part, source: fileSourceFromHandle(handle) } + }), + ) +} + /** * Poll cadence for the streamed video lifecycle. The server holds the request * open and polls the provider itself, so this is the rate at which @@ -172,6 +205,10 @@ function falL(model: Parameters[0]) { return falLiveVideo(model, { apiKey: requireByok(falByok) }) } +function falF() { + return falFiles({ apiKey: requireByok(falByok) }) +} + function grokI(model: Parameters[0]) { return createGrokImage(model, requireByok(grokByok)) } @@ -188,6 +225,10 @@ function geminiV(model: Parameters[0]) { return createGeminiVideo(model, requireByok(geminiByok)) } +function geminiF() { + return createGeminiFiles(requireByok(geminiByok)) +} + function byteplusI(model: Parameters[0]) { return createBytePlusImage(model, requireByok(byteplusByok)) } @@ -302,9 +343,14 @@ export const generateImageFn = createServerFn({ method: 'POST' }) }) } case 'gemini-3.1-flash-image': { + // Reference images are uploaded once via the Gemini Files API and + // referenced by handle (fileData.fileUri) rather than inlined as base64. return generateImage({ adapter: geminiI('gemini-3.1-flash-image'), - prompt: asImagePrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImagePrompt(data.prompt), + geminiF(), + ), numberOfImages: 1, size: '16:9_4K', }) @@ -312,7 +358,10 @@ export const generateImageFn = createServerFn({ method: 'POST' }) case 'gemini-3-pro-image': { return generateImage({ adapter: geminiI('gemini-3-pro-image'), - prompt: asImagePrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImagePrompt(data.prompt), + geminiF(), + ), numberOfImages: 1, size: '16:9_4K', }) @@ -386,7 +435,9 @@ interface VideoRequest { * browser's `useGenerateVideo` reads job id, status and result off these * chunks instead of running its own timer. */ -function videoStreamForModel(data: VideoRequest): AsyncIterable { +async function videoStreamForModel( + data: VideoRequest, +): Promise> { // Image-to-video models receive the start frame as a prompt part // (role: 'start_frame') — the fal adapter routes it to the endpoint's // start-image field. Text-to-video models take the text prompt only. @@ -479,14 +530,19 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { duration: adapter.snapDuration(6), }) } - // Image-to-video models + // Image-to-video models. The start frame is uploaded once to fal storage + // via the Files API (`falF()`) and referenced by its storage-URL + // handle, instead of being inlined as a base64 data: URI on the request. case 'fal-ai/kling-video/v3/pro/image-to-video': { const adapter = falV('fal-ai/kling-video/v3/pro/image-to-video') return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter, - prompt: asImageToVideoPrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImageToVideoPrompt(data.prompt), + falF(), + ), duration: adapter.snapDuration(5), modelOptions: { generate_audio: true, @@ -499,7 +555,10 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter, - prompt: asImageToVideoPrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImageToVideoPrompt(data.prompt), + falF(), + ), size: '16:9_1080p', duration: adapter.snapDuration(4), }) @@ -509,7 +568,10 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter: falV('xai/grok-imagine-video/image-to-video'), - prompt: asImageToVideoPrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImageToVideoPrompt(data.prompt), + falF(), + ), size: '16:9_720p', duration: 5, }) @@ -533,7 +595,10 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter, - prompt: asImageToVideoPrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImageToVideoPrompt(data.prompt), + falF(), + ), size: '16:9_2160p', duration: adapter.snapDuration(6), }) @@ -627,7 +692,9 @@ export const generateVideoFn = createServerFn({ method: 'POST' }) // before any stream exists, which surfaces as a plain server-function error // (the hook reports it through `error`) rather than a stream that opens only // to fail. - .handler(({ data }) => toServerSentEventsResponse(videoStreamForModel(data))) + .handler(async ({ data }) => + toServerSentEventsResponse(await videoStreamForModel(data)), + ) // ============================================================================ // Seedance Studio — BytePlus ModelArk direct (ARK_API_KEY, server-side only) diff --git a/examples/ts-react-ui-chatbot/src/components/chat/media-parts.tsx b/examples/ts-react-ui-chatbot/src/components/chat/media-parts.tsx index d10a57f218..64cdcd18e8 100644 --- a/examples/ts-react-ui-chatbot/src/components/chat/media-parts.tsx +++ b/examples/ts-react-ui-chatbot/src/components/chat/media-parts.tsx @@ -1,42 +1,59 @@ import type { PartProps } from '@tanstack/ai-react/ui' import type { chatOptions } from '@/chat/options' +/** + * Resolve a content source to something the browser can load. A + * `{ type: 'file' }` provider file handle is an opaque id (or an + * auth-gated provider URL), so there is nothing to render — return + * `undefined` and let the caller show a placeholder. + */ function sourceHref(source: { type: string - value: string + value?: string mimeType?: string -}): string { +}): string | undefined { + if (source.value === undefined) return undefined if (source.type === 'data') { return `data:${source.mimeType ?? 'application/octet-stream'};base64,${source.value}` } return source.value } +function UnrenderableSource({ label }: { label: string }) { + return ( +

+ {label} stored as a provider file handle +

+ ) +} + export function ImagePart({ part }: PartProps) { + const src = sourceHref(part.source) + if (!src) return return ( Trip photo ) } export function AudioPart({ part }: PartProps) { + const src = sourceHref(part.source) + if (!src) return return ( -