feat(ai-elevenlabs): add speech/audio/transcription adapters via official SDK

.changeset/elevenlabs-rest-adapters.md

@@ -0,0 +1,13 @@
+---
+'@tanstack/ai-elevenlabs': minor
+---
+
+feat: add REST adapters to @tanstack/ai-elevenlabs and migrate realtime to the renamed SDK
+
+Extends `@tanstack/ai-elevenlabs` (previously realtime-only) with three tree-shakeable REST adapters built on the official `@elevenlabs/elevenlabs-js` SDK (v2.44+):
+
+- `elevenlabsSpeech()` — text-to-speech on `eleven_v3`, `eleven_multilingual_v2`, `eleven_flash_*`, `eleven_turbo_*`
+- `elevenlabsAudio()` — music (`music_v1`, with structured composition plans) and sound effects (`eleven_text_to_sound_v2`/`v1`) via a single adapter that dispatches by model
+- `elevenlabsTranscription()` — Scribe v1/v2 speech-to-text with diarization, keyterm biasing, PII redaction, and word-level timestamps
+
+Also migrates the existing realtime adapter off the deprecated `@11labs/client` onto the renamed `@elevenlabs/client` package.

examples/ts-react-chat/src/lib/audio-providers.ts

@@ -6,7 +6,7 @@
  * and audio generation flows.
  */
 
-export type SpeechProviderId = 'openai' | 'gemini' | 'fal'
+export type SpeechProviderId = 'openai' | 'gemini' | 'fal' | 'elevenlabs'
 
 export interface SpeechProviderConfig {
   id: SpeechProviderId
@@ -55,9 +55,21 @@ export const SPEECH_PROVIDERS: ReadonlyArray<SpeechProviderConfig> = [
     ],
     placeholder: 'Enter text to synthesize with Fal Kokoro…',
   },
+  {
+    id: 'elevenlabs',
+    label: 'ElevenLabs',
+    model: 'eleven_multilingual_v2',
+    voices: [
+      { id: '21m00Tcm4TlvDq8ikWAM', label: 'Rachel' },
+      { id: 'AZnzlk1XvdvUeBnXmlld', label: 'Domi' },
+      { id: 'EXAVITQu4vr4xnSDxMaL', label: 'Bella' },
+      { id: 'pNInz6obpgDQGcFmaJgB', label: 'Adam' },
+    ],
+    placeholder: 'Enter text to synthesize with ElevenLabs…',
+  },
 ]
 
-export type TranscriptionProviderId = 'openai' | 'fal'
+export type TranscriptionProviderId = 'openai' | 'fal' | 'elevenlabs'
 
 export interface TranscriptionProviderConfig {
   id: TranscriptionProviderId
@@ -80,9 +92,21 @@ export const TRANSCRIPTION_PROVIDERS: ReadonlyArray<TranscriptionProviderConfig>
       model: 'fal-ai/whisper',
       description: 'Fal-hosted Whisper with word-level timestamps.',
     },
+    {
+      id: 'elevenlabs',
+      label: 'ElevenLabs Scribe',
+      model: 'scribe_v1',
+      description:
+        'ElevenLabs Scribe with diarization, keyterm biasing, and PII redaction.',
+    },
   ]
 
-export type AudioProviderId = 'gemini-lyria' | 'fal-audio' | 'fal-sfx'
+export type AudioProviderId =
+  | 'gemini-lyria'
+  | 'fal-audio'
+  | 'fal-sfx'
+  | 'elevenlabs-music'
+  | 'elevenlabs-sfx'
 
 export interface AudioProviderConfig {
   id: AudioProviderId
@@ -225,4 +249,70 @@ export const AUDIO_PROVIDERS: ReadonlyArray<AudioProviderConfig> = [
       },
     ],
   },
+  {
+    id: 'elevenlabs-music',
+    label: 'ElevenLabs Music',
+    model: 'music_v1',
+    models: [{ id: 'music_v1', label: 'Music v1' }],
+    description:
+      'ElevenLabs Music — free-form prompts or structured composition plans.',
+    placeholder: 'An upbeat synthwave track with driving drums and arpeggios',
+    defaultDuration: 15,
+    samplePrompts: [
+      {
+        label: 'Synthwave drive',
+        prompt:
+          'An upbeat synthwave track with driving drums, warm analog pads, and glittering arpeggios.',
+      },
+      {
+        label: 'Cinematic reveal',
+        prompt:
+          'A cinematic reveal score with soaring strings, low brass stabs, and a sudden timpani hit.',
+      },
+      {
+        label: 'Acoustic campfire',
+        prompt:
+          'A gentle acoustic campfire tune: fingerpicked guitar, soft harmonica, and distant crickets.',
+      },
+      {
+        label: 'Angry kazoo orchestra',
+        prompt:
+          'A furious kazoo orchestra performing an operatic aria about overdue library books.',
+      },
+    ],
+  },
+  {
+    id: 'elevenlabs-sfx',
+    label: 'ElevenLabs SFX',
+    model: 'eleven_text_to_sound_v2',
+    models: [
+      { id: 'eleven_text_to_sound_v2', label: 'Text-to-Sound v2' },
+      { id: 'eleven_text_to_sound_v1', label: 'Text-to-Sound v1' },
+    ],
+    description:
+      'ElevenLabs text-to-sound for short effects, 0.5–30 seconds per clip.',
+    placeholder: 'A whoosh followed by a deep bass impact',
+    defaultDuration: 5,
+    samplePrompts: [
+      {
+        label: 'Trailer whoosh',
+        prompt: 'A cinematic whoosh followed by a deep sub-bass impact.',
+      },
+      {
+        label: 'Sword unsheathe',
+        prompt:
+          'The crisp metallic ring of a sword being drawn from a leather scabbard.',
+      },
+      {
+        label: 'UI confirmation',
+        prompt:
+          'A short, satisfying UI confirmation tone with a subtle sparkle tail.',
+      },
+      {
+        label: 'Anxious toaster',
+        prompt:
+          'A small kitchen toaster having an anxiety attack: frantic clicks, steam, and a plaintive ding.',
+      },
+    ],
+  },
 ]

examples/ts-react-chat/src/lib/server-audio-adapters.ts

@@ -8,6 +8,11 @@
 import { openaiSpeech, openaiTranscription } from '@tanstack/ai-openai'
 import { geminiAudio, geminiSpeech } from '@tanstack/ai-gemini'
 import { falAudio, falSpeech, falTranscription } from '@tanstack/ai-fal'
+import {
+  elevenlabsAudio,
+  elevenlabsSpeech,
+  elevenlabsTranscription,
+} from '@tanstack/ai-elevenlabs'
 import type {
   AnyAudioAdapter,
   AnyTranscriptionAdapter,
@@ -40,6 +45,8 @@ export function buildSpeechAdapter(provider: SpeechProviderId): AnyTTSAdapter {
       return geminiSpeech(config.model as 'gemini-2.5-flash-preview-tts')
     case 'fal':
       return falSpeech(config.model)
+    case 'elevenlabs':
+      return elevenlabsSpeech(config.model)
   }
 }
 
@@ -52,6 +59,8 @@ export function buildTranscriptionAdapter(
       return openaiTranscription(config.model as 'whisper-1')
     case 'fal':
       return falTranscription(config.model)
+    case 'elevenlabs':
+      return elevenlabsTranscription(config.model)
   }
 }
 
@@ -69,6 +78,9 @@ export function buildAudioAdapter(
     case 'fal-audio':
     case 'fal-sfx':
       return falAudio(model)
+    case 'elevenlabs-music':
+    case 'elevenlabs-sfx':
+      return elevenlabsAudio(model)
   }
 }
 

examples/ts-react-chat/src/lib/server-fns.ts

@@ -17,12 +17,22 @@ import {
   buildTranscriptionAdapter,
 } from './server-audio-adapters'
 
-const SPEECH_PROVIDER_SCHEMA = z.enum(['openai', 'gemini', 'fal']).optional()
+const SPEECH_PROVIDER_SCHEMA = z
+  .enum(['openai', 'gemini', 'fal', 'elevenlabs'])
+  .optional()
 
-const TRANSCRIPTION_PROVIDER_SCHEMA = z.enum(['openai', 'fal']).optional()
+const TRANSCRIPTION_PROVIDER_SCHEMA = z
+  .enum(['openai', 'fal', 'elevenlabs'])
+  .optional()
 
 const AUDIO_PROVIDER_SCHEMA = z
-  .enum(['gemini-lyria', 'fal-audio', 'fal-sfx'])
+  .enum([
+    'gemini-lyria',
+    'fal-audio',
+    'fal-sfx',
+    'elevenlabs-music',
+    'elevenlabs-sfx',
+  ])
   .optional()
 
 // =============================================================================

examples/ts-react-chat/src/routes/api.generate.audio.ts

@@ -4,7 +4,13 @@ import { z } from 'zod'
 import { buildAudioAdapter } from '../lib/server-audio-adapters'
 
 const AUDIO_PROVIDER_SCHEMA = z
-  .enum(['gemini-lyria', 'fal-audio', 'fal-sfx'])
+  .enum([
+    'gemini-lyria',
+    'fal-audio',
+    'fal-sfx',
+    'elevenlabs-music',
+    'elevenlabs-sfx',
+  ])
   .optional()
 
 const AUDIO_BODY_SCHEMA = z.object({

examples/ts-react-chat/src/routes/api.generate.speech.ts

@@ -3,7 +3,9 @@ import { generateSpeech, toServerSentEventsResponse } from '@tanstack/ai'
 import { z } from 'zod'
 import { buildSpeechAdapter } from '../lib/server-audio-adapters'
 
-const SPEECH_PROVIDER_SCHEMA = z.enum(['openai', 'gemini', 'fal']).optional()
+const SPEECH_PROVIDER_SCHEMA = z
+  .enum(['openai', 'gemini', 'fal', 'elevenlabs'])
+  .optional()
 
 const SPEECH_BODY_SCHEMA = z.object({
   text: z.string().min(1),

examples/ts-react-chat/src/routes/api.transcribe.ts

@@ -3,7 +3,9 @@ import { generateTranscription, toServerSentEventsResponse } from '@tanstack/ai'
 import { z } from 'zod'
 import { buildTranscriptionAdapter } from '../lib/server-audio-adapters'
 
-const TRANSCRIPTION_PROVIDER_SCHEMA = z.enum(['openai', 'fal']).optional()
+const TRANSCRIPTION_PROVIDER_SCHEMA = z
+  .enum(['openai', 'fal', 'elevenlabs'])
+  .optional()
 
 const TRANSCRIBE_BODY_SCHEMA = z.object({
   audio: z.string().min(1),

examples/ts-react-chat/vite.config.ts

@@ -7,9 +7,24 @@ import { nitroV2Plugin } from '@tanstack/nitro-v2-vite-plugin'
 import { devtools } from '@tanstack/devtools-vite'
 
 const config = defineConfig({
+  // Server-side only fix. @elevenlabs/elevenlabs-js ships a top-level
+  // `function getHeader(…)` that collides with h3's auto-imported
+  // `getHeader` when vite inlines it into the SSR bundle. The SDK is
+  // only imported by server-side adapter factories (see
+  // `src/lib/server-audio-adapters.ts`), so tree-shaking already keeps
+  // it out of the client bundle — this option only affects the SSR
+  // build, where we want the SDK resolved at runtime via require()
+  // instead of inlined into the rollup chunk.
+  ssr: {
+    external: ['@elevenlabs/elevenlabs-js'],
+  },
   plugins: [
     devtools(),
-    nitroV2Plugin(),
+    nitroV2Plugin({
+      externals: {
+        external: ['@elevenlabs/elevenlabs-js'],
+      },
+    }),
     // this is the plugin that enables path aliases
     viteTsConfigPaths({
       projects: ['./tsconfig.json'],

package.json

@@ -37,6 +37,7 @@
     "build:all": "nx run-many --targets=build --exclude=examples/**",
     "watch": "pnpm run build:all && env NX_DAEMON=true nx watch --all -- pnpm run build:all",
     "dev": "pnpm run watch",
+    "dev:chat": "pnpm --filter ts-react-chat dev",
     "format": "prettier --experimental-cli --ignore-unknown '**/*' --write",
     "generate-docs": "node scripts/generate-docs.ts && pnpm run copy:readme",
     "generate:models": "pnpm generate:models:fetch && pnpm regenerate:models && tsx scripts/sync-provider-models.ts && pnpm format",

packages/typescript/ai-elevenlabs/package.json

@@ -15,7 +15,14 @@
     "voice",
     "realtime",
     "tanstack",
-    "adapter"
+    "adapter",
+    "tts",
+    "text-to-speech",
+    "audio-generation",
+    "music",
+    "sound-effects",
+    "transcription",
+    "speech-to-text"
   ],
   "type": "module",
   "module": "./dist/esm/index.js",
@@ -41,7 +48,8 @@
     "test:types": "tsc"
   },
   "dependencies": {
-    "@11labs/client": "^0.2.0"
+    "@elevenlabs/client": "^1.3.1",
+    "@elevenlabs/elevenlabs-js": "^2.44.0"
   },
   "peerDependencies": {
     "@tanstack/ai": "workspace:^",