feat: add realtime chat support with OpenAI and ElevenLabs adapters

.changeset/realtime-chat.md

@@ -0,0 +1,11 @@
+---
+'@tanstack/ai': minor
+'@tanstack/ai-client': minor
+'@tanstack/ai-openai': minor
+'@tanstack/ai-elevenlabs': minor
+'@tanstack/ai-react': minor
+---
+
+feat: add realtime chat support with OpenAI and ElevenLabs adapters
+
+Adds realtime voice/text chat capabilities including a provider-agnostic realtime client, OpenAI Realtime API adapter, ElevenLabs conversational AI adapter, React `useRealtimeChat` hook, and shared realtime types across the core, client, and framework packages.

examples/ts-react-chat/.env.example

@@ -1,3 +1,11 @@
 # OpenAI API Key
 # Get yours at: https://platform.openai.com/api-keys
-OPENAI_API_KEY=sk-...
+OPENAI_API_KEY=sk-...
+
+# ElevenLabs API Key (for realtime voice)
+# Get yours at: https://elevenlabs.io/app/settings/api-keys
+ELEVENLABS_API_KEY=xi-...
+
+# ElevenLabs Agent ID (for realtime voice)
+# Create an agent at: https://elevenlabs.io/app/conversational-ai
+ELEVENLABS_AGENT_ID=...

examples/ts-react-chat/package.json

@@ -13,6 +13,7 @@
     "@tanstack/ai": "workspace:*",
     "@tanstack/ai-anthropic": "workspace:*",
     "@tanstack/ai-client": "workspace:*",
+    "@tanstack/ai-elevenlabs": "workspace:*",
     "@tanstack/ai-gemini": "workspace:*",
     "@tanstack/ai-grok": "workspace:*",
     "@tanstack/ai-ollama": "workspace:*",

examples/ts-react-chat/src/components/AudioSparkline.tsx

@@ -0,0 +1,81 @@
+import { useEffect, useRef } from 'react'
+
+export function AudioSparkline({
+  getData,
+  color,
+  label,
+}: {
+  getData: () => Uint8Array
+  color: string
+  label: string
+}) {
+  const canvasRef = useRef<HTMLCanvasElement>(null)
+  const animationRef = useRef<number | null>(null)
+
+  useEffect(() => {
+    const canvas = canvasRef.current
+    if (!canvas) return
+
+    const ctx = canvas.getContext('2d')
+    if (!ctx) return
+
+    function draw() {
+      const data = getData()
+      const width = canvas!.width
+      const height = canvas!.height
+
+      ctx!.fillStyle = '#1f2937'
+      ctx!.fillRect(0, 0, width, height)
+
+      ctx!.strokeStyle = color
+      ctx!.lineWidth = 1
+      ctx!.beginPath()
+
+      const step = Math.max(1, Math.floor(data.length / width))
+
+      for (let i = 0; i < width; i++) {
+        const dataIndex = Math.min(i * step, data.length - 1)
+        const value = data[dataIndex] ?? 128
+        const y = height - (value / 255) * height
+
+        if (i === 0) {
+          ctx!.moveTo(i, y)
+        } else {
+          ctx!.lineTo(i, y)
+        }
+      }
+
+      ctx!.stroke()
+
+      ctx!.strokeStyle = '#4b5563'
+      ctx!.setLineDash([2, 2])
+      ctx!.beginPath()
+      ctx!.moveTo(0, height / 2)
+      ctx!.lineTo(width, height / 2)
+      ctx!.stroke()
+      ctx!.setLineDash([])
+
+      animationRef.current = requestAnimationFrame(draw)
+    }
+
+    draw()
+
+    return () => {
+      if (animationRef.current) {
+        cancelAnimationFrame(animationRef.current)
+      }
+    }
+  }, [getData, color])
+
+  return (
+    <div className="flex items-center gap-2">
+      <span className="text-xs text-gray-500 w-12">{label}</span>
+      <canvas
+        ref={canvasRef}
+        width={200}
+        height={40}
+        className="rounded border border-gray-700"
+      />
+    </div>
+  )
+}

examples/ts-react-chat/src/components/Header.tsx

@@ -1,7 +1,7 @@
 import { Link } from '@tanstack/react-router'
 
 import { useState } from 'react'
-import { Guitar, Home, Menu, X } from 'lucide-react'
+import { Guitar, Home, Menu, Mic, X } from 'lucide-react'
 
 export default function Header() {
   const [isOpen, setIsOpen] = useState(false)
@@ -71,6 +71,19 @@ export default function Header() {
             <Guitar size={20} />
             <span className="font-medium">Guitar Demo</span>
           </Link>
+
+          <Link
+            to="/realtime"
+            onClick={() => setIsOpen(false)}
+            className="flex items-center gap-3 p-3 rounded-lg hover:bg-gray-800 transition-colors mb-2"
+            activeProps={{
+              className:
+                'flex items-center gap-3 p-3 rounded-lg bg-cyan-600 hover:bg-cyan-700 transition-colors mb-2',
+            }}
+          >
+            <Mic size={20} />
+            <span className="font-medium">Voice Chat (Realtime)</span>
+          </Link>
         </nav>
       </aside>
     </>

examples/ts-react-chat/src/lib/realtime-tools.ts

@@ -0,0 +1,167 @@
+import { toolDefinition } from '@tanstack/ai'
+import { z } from 'zod'
+
+// Tool to get current time - useful for voice assistants
+export const getCurrentTimeToolDef = toolDefinition({
+  name: 'getCurrentTime',
+  description:
+    'Get the current date and time. Use this when the user asks what time it is or the current date.',
+  inputSchema: z.object({
+    timezone: z
+      .string()
+      .optional()
+      .describe('Optional timezone like "America/New_York" or "Europe/London"'),
+  }),
+  outputSchema: z.object({
+    time: z.string(),
+    date: z.string(),
+    timezone: z.string(),
+  }),
+})
+
+// Tool to get weather - common voice assistant use case
+export const getWeatherToolDef = toolDefinition({
+  name: 'getWeather',
+  description:
+    'Get the current weather for a location. Use this when the user asks about the weather.',
+  inputSchema: z.object({
+    location: z
+      .string()
+      .describe(
+        'The city and state/country, e.g. "San Francisco, CA" or "London, UK"',
+      ),
+  }),
+  outputSchema: z.object({
+    location: z.string(),
+    temperature: z.number(),
+    unit: z.string(),
+    condition: z.string(),
+    humidity: z.number(),
+  }),
+})
+
+// Tool to set a reminder - demonstrates user interaction
+export const setReminderToolDef = toolDefinition({
+  name: 'setReminder',
+  description:
+    'Set a reminder for the user. Use this when the user asks to be reminded about something.',
+  inputSchema: z.object({
+    message: z.string().describe('What to remind the user about'),
+    inMinutes: z.number().describe('How many minutes from now to remind'),
+  }),
+  outputSchema: z.object({
+    success: z.boolean(),
+    message: z.string(),
+    remindAt: z.string(),
+  }),
+})
+
+// Tool to search knowledge base - useful for assistants with specific knowledge
+export const searchKnowledgeToolDef = toolDefinition({
+  name: 'searchKnowledge',
+  description:
+    'Search a knowledge base for information. Use this to find specific facts or documentation.',
+  inputSchema: z.object({
+    query: z.string().describe('The search query'),
+  }),
+  outputSchema: z.object({
+    results: z.array(
+      z.object({
+        title: z.string(),
+        snippet: z.string(),
+      }),
+    ),
+  }),
+})
+
+// Client-side implementation of getCurrentTime
+export const getCurrentTimeClient = getCurrentTimeToolDef.client(
+  ({ timezone }) => {
+    const now = new Date()
+    const tz = timezone || Intl.DateTimeFormat().resolvedOptions().timeZone
+
+    return {
+      time: now.toLocaleTimeString('en-US', { timeZone: tz }),
+      date: now.toLocaleDateString('en-US', {
+        weekday: 'long',
+        year: 'numeric',
+        month: 'long',
+        day: 'numeric',
+        timeZone: tz,
+      }),
+      timezone: tz,
+    }
+  },
+)
+
+// Client-side implementation of getWeather (mock data for demo)
+export const getWeatherClient = getWeatherToolDef.client(({ location }) => {
+  // Mock weather data for demo purposes
+  const conditions = ['Sunny', 'Partly Cloudy', 'Cloudy', 'Rainy', 'Snowy']
+  const randomCondition =
+    conditions[Math.floor(Math.random() * conditions.length)]!
+  const randomTemp = Math.floor(Math.random() * 30) + 50 // 50-80°F
+  const randomHumidity = Math.floor(Math.random() * 50) + 30 // 30-80%
+
+  return {
+    location,
+    temperature: randomTemp,
+    unit: 'F',
+    condition: randomCondition,
+    humidity: randomHumidity,
+  }
+})
+
+// Client-side implementation of setReminder
+export const setReminderClient = setReminderToolDef.client(
+  ({ message, inMinutes }) => {
+    const remindAt = new Date(Date.now() + inMinutes * 60 * 1000)
+
+    // In a real app, you'd schedule a notification here
+    console.log(
+      `[Reminder] Will remind about "${message}" at ${remindAt.toLocaleTimeString()}`,
+    )
+
+    // For demo purposes, show an alert after the specified time
+    setTimeout(
+      () => {
+        alert(`Reminder: ${message}`)
+      },
+      inMinutes * 60 * 1000,
+    )
+
+    return {
+      success: true,
+      message: `Reminder set: "${message}"`,
+      remindAt: remindAt.toLocaleTimeString(),
+    }
+  },
+)
+
+// Client-side implementation of searchKnowledge (mock data for demo)
+export const searchKnowledgeClient = searchKnowledgeToolDef.client(
+  ({ query }) => {
+    // Mock search results for demo
+    const mockResults = [
+      {
+        title: `Result for: ${query}`,
+        snippet: `This is a mock search result for the query "${query}". In a real application, this would return actual search results from a knowledge base.`,
+      },
+      {
+        title: 'Additional Information',
+        snippet:
+          'More relevant information would appear here based on your search query.',
+      },
+    ]
+
+    return { results: mockResults }
+  },
+)
+
+// Export all client tools as an array for easy use
+export const realtimeClientTools = [
+  getCurrentTimeClient,
+  getWeatherClient,
+  setReminderClient,
+  searchKnowledgeClient,
+] as const

examples/ts-react-chat/src/lib/use-realtime.ts

@@ -0,0 +1,78 @@
+import { createServerFn } from '@tanstack/react-start'
+import { realtimeToken } from '@tanstack/ai'
+import { useRealtimeChat } from '@tanstack/ai-react'
+import { openaiRealtime, openaiRealtimeToken } from '@tanstack/ai-openai'
+import {
+  elevenlabsRealtime,
+  elevenlabsRealtimeToken,
+} from '@tanstack/ai-elevenlabs'
+import { realtimeClientTools } from '@/lib/realtime-tools'
+
+type Provider = 'openai' | 'elevenlabs'
+
+const getRealtimeTokenFn = createServerFn({ method: 'POST' })
+  .inputValidator((data: { provider: Provider; agentId?: string }) => {
+    if (!data.provider) throw new Error('Provider is required')
+    return data
+  })
+  .handler(async ({ data }) => {
+    if (data.provider === 'openai') {
+      return realtimeToken({
+        adapter: openaiRealtimeToken({
+          model: 'gpt-4o-realtime-preview',
+        }),
+      })
+    }
+
+    if (data.provider === 'elevenlabs') {
+      const agentId = data.agentId || process.env.ELEVENLABS_AGENT_ID
+      if (!agentId) {
+        throw new Error(
+          'ElevenLabs agent ID is required. Set ELEVENLABS_AGENT_ID or pass agentId in request body.',
+        )
+      }
+      return realtimeToken({
+        adapter: elevenlabsRealtimeToken({ agentId }),
+      })
+    }
+
+    throw new Error(`Unknown provider: ${data.provider}`)
+  })
+
+export function useRealtime({
+  provider,
+  agentId,
+}: {
+  provider: Provider
+  agentId: string
+}) {
+  const adapter =
+    provider === 'openai' ? openaiRealtime() : elevenlabsRealtime()
+
+  return useRealtimeChat({
+    getToken: () =>
+      getRealtimeTokenFn({
+        data: {
+          provider,
+          ...(provider === 'elevenlabs' && agentId ? { agentId } : {}),
+        },
+      }),
+    adapter,
+    instructions: `You are a helpful, friendly voice assistant with access to several tools.
+
+You can:
+- Tell the user the current time and date (getCurrentTime)
+- Get weather information for any location (getWeather)
+- Set reminders for the user (setReminder)
+- Search a knowledge base for information (searchKnowledge)
+
+Keep your responses concise and conversational since this is a voice interface.
+When using tools, briefly explain what you're doing and then share the results naturally.
+Be friendly and engaging!`,
+    voice: 'alloy',
+    tools: realtimeClientTools,
+    onError: (err) => {
+      console.error('Realtime error:', err)
+    },
+  })
+}