feat(ai-grok): audio, speech, and realtime adapters + example wiring

.changeset/grok-audio-speech-support.md

@@ -0,0 +1,13 @@
+---
+'@tanstack/ai-grok': minor
+---
+
+feat(ai-grok): add audio and speech adapters for xAI
+
+Add three new tree-shakeable adapters that wrap xAI's audio APIs:
+
+- `grokSpeech` / `createGrokSpeech` — text-to-speech via `POST /v1/tts`. Supports the 5 xAI voices (`eve`, `ara`, `rex`, `sal`, `leo`), MP3/WAV/PCM/μ-law/A-law codecs, and the `language`, `sample_rate`, `bit_rate`, `optimize_streaming_latency`, `text_normalization` provider options.
+- `grokTranscription` / `createGrokTranscription` — speech-to-text via `POST /v1/stt`. Passes through `language`, `diarize`, `multichannel`, `channels`, `audio_format`, and `sample_rate`; maps xAI's word-level timestamps to `TranscriptionResult.words`.
+- `grokRealtime` / `grokRealtimeToken` — Voice Agent (realtime) adapter for `wss://api.x.ai/v1/realtime` with ephemeral tokens via `/v1/realtime/client_secrets`. Supports the `grok-voice-fast-1.0` and `grok-voice-think-fast-1.0` models.
+
+New model identifier exports: `GROK_TTS_MODELS`, `GROK_TRANSCRIPTION_MODELS`, `GROK_REALTIME_MODELS` and their corresponding types.

examples/ts-react-chat/src/lib/audio-providers.ts

@@ -6,7 +6,7 @@
  * and audio generation flows.
  */
 
-export type SpeechProviderId = 'openai' | 'gemini' | 'fal'
+export type SpeechProviderId = 'openai' | 'gemini' | 'fal' | 'grok'
 
 export interface SpeechProviderConfig {
   id: SpeechProviderId
@@ -55,9 +55,22 @@ export const SPEECH_PROVIDERS: ReadonlyArray<SpeechProviderConfig> = [
     ],
     placeholder: 'Enter text to synthesize with Fal Kokoro…',
   },
+  {
+    id: 'grok',
+    label: 'Grok TTS',
+    model: 'grok-tts',
+    voices: [
+      { id: 'eve', label: 'Eve' },
+      { id: 'ara', label: 'Ara' },
+      { id: 'rex', label: 'Rex' },
+      { id: 'sal', label: 'Sal' },
+      { id: 'leo', label: 'Leo' },
+    ],
+    placeholder: 'Enter text for Grok speech…',
+  },
 ]
 
-export type TranscriptionProviderId = 'openai' | 'fal'
+export type TranscriptionProviderId = 'openai' | 'fal' | 'grok'
 
 export interface TranscriptionProviderConfig {
   id: TranscriptionProviderId
@@ -80,6 +93,12 @@ export const TRANSCRIPTION_PROVIDERS: ReadonlyArray<TranscriptionProviderConfig>
       model: 'fal-ai/whisper',
       description: 'Fal-hosted Whisper with word-level timestamps.',
     },
+    {
+      id: 'grok',
+      label: 'Grok STT',
+      model: 'grok-stt',
+      description: 'xAI speech-to-text with word-level timestamps.',
+    },
   ]
 
 export type AudioProviderId = 'gemini-lyria' | 'fal-audio' | 'fal-sfx'

examples/ts-react-chat/src/lib/server-audio-adapters.ts

@@ -8,6 +8,7 @@
 import { openaiSpeech, openaiTranscription } from '@tanstack/ai-openai'
 import { geminiAudio, geminiSpeech } from '@tanstack/ai-gemini'
 import { falAudio, falSpeech, falTranscription } from '@tanstack/ai-fal'
+import { grokSpeech, grokTranscription } from '@tanstack/ai-grok'
 import type {
   AnyAudioAdapter,
   AnyTranscriptionAdapter,
@@ -40,6 +41,8 @@ export function buildSpeechAdapter(provider: SpeechProviderId): AnyTTSAdapter {
       return geminiSpeech(config.model as 'gemini-2.5-flash-preview-tts')
     case 'fal':
       return falSpeech(config.model)
+    case 'grok':
+      return grokSpeech(config.model as 'grok-tts')
   }
 }
 
@@ -52,6 +55,8 @@ export function buildTranscriptionAdapter(
       return openaiTranscription(config.model as 'whisper-1')
     case 'fal':
       return falTranscription(config.model)
+    case 'grok':
+      return grokTranscription(config.model as 'grok-stt')
   }
 }
 

examples/ts-react-chat/src/lib/server-fns.ts

@@ -17,9 +17,13 @@ import {
   buildTranscriptionAdapter,
 } from './server-audio-adapters'
 
-const SPEECH_PROVIDER_SCHEMA = z.enum(['openai', 'gemini', 'fal']).optional()
+const SPEECH_PROVIDER_SCHEMA = z
+  .enum(['openai', 'gemini', 'fal', 'grok'])
+  .optional()
 
-const TRANSCRIPTION_PROVIDER_SCHEMA = z.enum(['openai', 'fal']).optional()
+const TRANSCRIPTION_PROVIDER_SCHEMA = z
+  .enum(['openai', 'fal', 'grok'])
+  .optional()
 
 const AUDIO_PROVIDER_SCHEMA = z
   .enum(['gemini-lyria', 'fal-audio', 'fal-sfx'])

examples/ts-react-chat/src/lib/use-realtime.ts

@@ -6,9 +6,10 @@ import {
   elevenlabsRealtime,
   elevenlabsRealtimeToken,
 } from '@tanstack/ai-elevenlabs'
+import { grokRealtime, grokRealtimeToken } from '@tanstack/ai-grok'
 import { realtimeClientTools } from '@/lib/realtime-tools'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'grok'
 
 const getRealtimeTokenFn = createServerFn({ method: 'POST' })
   .inputValidator((data: { provider: Provider; agentId?: string }) => {
@@ -36,27 +37,43 @@ const getRealtimeTokenFn = createServerFn({ method: 'POST' })
       })
     }
 
+    if (data.provider === 'grok') {
+      return realtimeToken({
+        adapter: grokRealtimeToken({ model: 'grok-voice-fast-1.0' }),
+      })
+    }
+
     throw new Error(`Unknown provider: ${data.provider}`)
   })
 
+function adapterForProvider(provider: Provider) {
+  switch (provider) {
+    case 'openai':
+      return openaiRealtime()
+    case 'elevenlabs':
+      return elevenlabsRealtime()
+    case 'grok':
+      return grokRealtime()
+  }
+}
+
 export function useRealtime({
   provider,
   agentId,
+  voice,
   outputModalities,
   temperature,
   maxOutputTokens,
   semanticEagerness,
 }: {
   provider: Provider
   agentId: string
+  voice?: string
   outputModalities?: Array<'audio' | 'text'>
   temperature?: number
   maxOutputTokens?: number | 'inf'
   semanticEagerness?: 'low' | 'medium' | 'high'
 }) {
-  const adapter =
-    provider === 'openai' ? openaiRealtime() : elevenlabsRealtime()
-
   return useRealtimeChat({
     getToken: () =>
       getRealtimeTokenFn({
@@ -65,7 +82,7 @@ export function useRealtime({
           ...(provider === 'elevenlabs' && agentId ? { agentId } : {}),
         },
       }),
-    adapter,
+    adapter: adapterForProvider(provider),
     instructions: `You are a helpful, friendly voice assistant with access to several tools.
 
 You can:
@@ -78,7 +95,7 @@ Keep your responses concise and conversational since this is a voice interface.
 When using tools, briefly explain what you're doing and then share the results naturally.
 If the user sends an image, describe what you see and answer any questions about it.
 Be friendly and engaging!`,
-    voice: 'alloy',
+    voice: voice ?? (provider === 'grok' ? 'eve' : 'alloy'),
     tools: realtimeClientTools,
     outputModalities,
     temperature,

examples/ts-react-chat/src/routes/api.generate.speech.ts

@@ -3,7 +3,9 @@ import { generateSpeech, toServerSentEventsResponse } from '@tanstack/ai'
 import { z } from 'zod'
 import { buildSpeechAdapter } from '../lib/server-audio-adapters'
 
-const SPEECH_PROVIDER_SCHEMA = z.enum(['openai', 'gemini', 'fal']).optional()
+const SPEECH_PROVIDER_SCHEMA = z
+  .enum(['openai', 'gemini', 'fal', 'grok'])
+  .optional()
 
 const SPEECH_BODY_SCHEMA = z.object({
   text: z.string().min(1),

examples/ts-react-chat/src/routes/api.transcribe.ts

@@ -3,7 +3,9 @@ import { generateTranscription, toServerSentEventsResponse } from '@tanstack/ai'
 import { z } from 'zod'
 import { buildTranscriptionAdapter } from '../lib/server-audio-adapters'
 
-const TRANSCRIPTION_PROVIDER_SCHEMA = z.enum(['openai', 'fal']).optional()
+const TRANSCRIPTION_PROVIDER_SCHEMA = z
+  .enum(['openai', 'fal', 'grok'])
+  .optional()
 
 const TRANSCRIBE_BODY_SCHEMA = z.object({
   audio: z.string().min(1),

examples/ts-react-chat/src/routes/realtime.tsx

@@ -13,14 +13,18 @@ import {
 import { AudioSparkline } from '@/components/AudioSparkline'
 import { useRealtime } from '@/lib/use-realtime'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'grok'
 type OutputMode = 'audio+text' | 'text-only' | 'audio-only'
 
 const PROVIDER_OPTIONS: Array<{ value: Provider; label: string }> = [
   { value: 'openai', label: 'OpenAI Realtime' },
   { value: 'elevenlabs', label: 'ElevenLabs' },
+  { value: 'grok', label: 'Grok Voice Agent' },
 ]
 
+const GROK_VOICES = ['eve', 'ara', 'rex', 'sal', 'leo'] as const
+type GrokVoice = (typeof GROK_VOICES)[number]
+
 const OUTPUT_MODE_OPTIONS: Array<{ value: OutputMode; label: string }> = [
   { value: 'audio+text', label: 'Audio + Text' },
   { value: 'text-only', label: 'Text Only' },
@@ -45,6 +49,7 @@ function outputModeToModalities(
 function RealtimePage() {
   const [provider, setProvider] = useState<Provider>('openai')
   const [agentId, setAgentId] = useState('')
+  const [grokVoice, setGrokVoice] = useState<GrokVoice>('eve')
   const [textInput, setTextInput] = useState('')
   const [outputMode, setOutputMode] = useState<OutputMode>('audio+text')
   const [temperature, setTemperature] = useState(0.8)
@@ -73,6 +78,7 @@ function RealtimePage() {
   } = useRealtime({
     provider,
     agentId,
+    voice: provider === 'grok' ? grokVoice : undefined,
     outputModalities: outputModeToModalities(outputMode),
     temperature,
     semanticEagerness,
@@ -195,8 +201,29 @@ function RealtimePage() {
                 </div>
               )}
 
-              {/* Output mode selector (OpenAI only) */}
-              {provider === 'openai' && (
+              {/* Grok voice selector */}
+              {provider === 'grok' && (
+                <div>
+                  <label className="text-sm text-gray-400 mb-1 block">
+                    Voice
+                  </label>
+                  <select
+                    value={grokVoice}
+                    onChange={(e) => setGrokVoice(e.target.value as GrokVoice)}
+                    disabled={status !== 'idle'}
+                    className="rounded-lg border border-orange-500/20 bg-gray-900 px-3 py-2 text-sm text-white focus:outline-none focus:ring-2 focus:ring-orange-500/50 disabled:opacity-50"
+                  >
+                    {GROK_VOICES.map((v) => (
+                      <option key={v} value={v}>
+                        {v}
+                      </option>
+                    ))}
+                  </select>
+                </div>
+              )}
+
+              {/* Output mode selector (OpenAI-compatible realtime) */}
+              {(provider === 'openai' || provider === 'grok') && (
                 <div>
                   <label className="text-sm text-gray-400 mb-1 block">
                     Output
@@ -219,7 +246,7 @@ function RealtimePage() {
               )}
 
               {/* Temperature slider */}
-              {provider === 'openai' && (
+              {(provider === 'openai' || provider === 'grok') && (
                 <div>
                   <label className="text-sm text-gray-400 mb-1 block">
                     Temp: {temperature.toFixed(1)}
@@ -238,7 +265,7 @@ function RealtimePage() {
               )}
 
               {/* Semantic eagerness */}
-              {provider === 'openai' && (
+              {(provider === 'openai' || provider === 'grok') && (
                 <div>
                   <label className="text-sm text-gray-400 mb-1 block">
                     Eagerness
@@ -275,7 +302,7 @@ function RealtimePage() {
         </div>
 
         {/* Tools indicator */}
-        {provider === 'openai' && (
+        {(provider === 'openai' || provider === 'grok') && (
           <div className="border-b border-orange-500/10 bg-gray-800/50 px-4 py-2">
             <div className="flex items-center gap-2 text-xs text-gray-400">
               <Wrench className="w-3 h-3" />
@@ -422,8 +449,8 @@ function RealtimePage() {
                 placeholder="Type a message..."
                 className="flex-1 rounded-lg border border-orange-500/20 bg-gray-800 px-4 py-2 text-sm text-white placeholder-gray-500 focus:outline-none focus:ring-2 focus:ring-orange-500/50"
               />
-              {/* Image upload button (OpenAI only) */}
-              {provider === 'openai' && (
+              {/* Image upload button (OpenAI-compatible realtime) */}
+              {(provider === 'openai' || provider === 'grok') && (
                 <>
                   <input
                     ref={imageInputRef}

packages/typescript/ai-grok/package.json

@@ -47,11 +47,15 @@
     "openai": "^6.9.1"
   },
   "devDependencies": {
+    "@tanstack/ai": "workspace:*",
+    "@tanstack/ai-client": "workspace:*",
     "@vitest/coverage-v8": "4.0.14",
-    "vite": "^7.2.7"
+    "vite": "^7.2.7",
+    "zod": "^4.2.0"
   },
   "peerDependencies": {
     "@tanstack/ai": "workspace:^",
+    "@tanstack/ai-client": "workspace:^",
     "zod": "^4.0.0"
   }
 }