Feat(ai-gemini): Gemini Realtime Adapter

.changeset/huge-lizards-admire.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/ai-gemini': minor
+---
+
+Added Gemini Realtime Adapter

examples/ts-react-chat/src/lib/use-realtime.ts

@@ -6,12 +6,15 @@ import {
   elevenlabsRealtime,
   elevenlabsRealtimeToken,
 } from '@tanstack/ai-elevenlabs'
+import { geminiRealtime, geminiRealtimeToken } from '@tanstack/ai-gemini'
+import type { GeminiRealtimeVoice } from "@tanstack/ai-gemini"
+import type { OpenAIRealtimeVoice } from "@tanstack/ai-openai"
 import { realtimeClientTools } from '@/lib/realtime-tools'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'gemini'
 
 const getRealtimeTokenFn = createServerFn({ method: 'POST' })
-  .inputValidator((data: { provider: Provider; agentId?: string }) => {
+  .inputValidator((data: { provider?: Provider; agentId?: string }) => {
     if (!data.provider) throw new Error('Provider is required')
     return data
   })
@@ -24,6 +27,12 @@ const getRealtimeTokenFn = createServerFn({ method: 'POST' })
       })
     }
 
+    if (data.provider === 'gemini') {
+      return realtimeToken({
+        adapter: geminiRealtimeToken(),
+      })
+    }
+
     if (data.provider === 'elevenlabs') {
       const agentId = data.agentId || process.env.ELEVENLABS_AGENT_ID
       if (!agentId) {
@@ -46,16 +55,22 @@ export function useRealtime({
   temperature,
   maxOutputTokens,
   semanticEagerness,
+  voice
 }: {
   provider: Provider
   agentId: string
   outputModalities?: Array<'audio' | 'text'>
   temperature?: number
   maxOutputTokens?: number | 'inf'
   semanticEagerness?: 'low' | 'medium' | 'high'
+  voice?: OpenAIRealtimeVoice | GeminiRealtimeVoice
 }) {
   const adapter =
-    provider === 'openai' ? openaiRealtime() : elevenlabsRealtime()
+    provider === 'openai'
+      ? openaiRealtime()
+      : provider === 'gemini'
+        ? geminiRealtime()
+        : elevenlabsRealtime()
 
   return useRealtimeChat({
     getToken: () =>
@@ -78,7 +93,7 @@ Keep your responses concise and conversational since this is a voice interface.
 When using tools, briefly explain what you're doing and then share the results naturally.
 If the user sends an image, describe what you see and answer any questions about it.
 Be friendly and engaging!`,
-    voice: 'alloy',
+    voice: voice || provider === 'gemini' ? 'Puck' : 'alloy',
     tools: realtimeClientTools,
     outputModalities,
     temperature,

examples/ts-react-chat/src/routes/realtime.tsx

@@ -13,11 +13,12 @@ import {
 import { AudioSparkline } from '@/components/AudioSparkline'
 import { useRealtime } from '@/lib/use-realtime'
 
-type Provider = 'openai' | 'elevenlabs'
+type Provider = 'openai' | 'elevenlabs' | 'gemini'
 type OutputMode = 'audio+text' | 'text-only' | 'audio-only'
 
 const PROVIDER_OPTIONS: Array<{ value: Provider; label: string }> = [
   { value: 'openai', label: 'OpenAI Realtime' },
+  { value: 'gemini', label: 'Google Gemini' },
   { value: 'elevenlabs', label: 'ElevenLabs' },
 ]
 
@@ -275,7 +276,7 @@ function RealtimePage() {
         </div>
 
         {/* Tools indicator */}
-        {provider === 'openai' && (
+        {(provider === 'openai' || provider === 'gemini') && (
           <div className="border-b border-orange-500/10 bg-gray-800/50 px-4 py-2">
             <div className="flex items-center gap-2 text-xs text-gray-400">
               <Wrench className="w-3 h-3" />

packages/typescript/ai-client/src/realtime-client.ts

@@ -100,8 +100,30 @@ export class RealtimeClient {
         this.clientTools.size > 0
           ? Array.from(this.clientTools.values())
           : undefined
+
+      const {
+        instructions,
+        voice,
+        vadMode,
+        outputModalities,
+        temperature,
+        maxOutputTokens,
+        semanticEagerness,
+        providerOptions,
+      } = this.options
+
       this.connection = await this.options.adapter.connect(
-        this.token,
+        this.token, 
+        {
+          instructions,
+          voice,
+          vadMode,
+          outputModalities,
+          temperature,
+          maxOutputTokens,
+          semanticEagerness,
+          providerOptions,
+        },
         toolsList,
       )
 
@@ -472,6 +494,18 @@ export class RealtimeClient {
         this.options.onError?.(error)
       }),
     )
+
+    this.unsubscribers.push(
+      this.connection.on('go_away', ({ timeLeft }) => {
+        this.options.onGoAway?.(timeLeft)
+      }),
+    )
+
+    this.unsubscribers.push(
+      this.connection.on('usage', (usage) => {
+        this.options.onUsage?.(usage)
+      }),
+    )
   }
 
   private applySessionConfig(): void {
@@ -500,12 +534,12 @@ export class RealtimeClient {
 
     const toolsConfig = tools
       ? Array.from(this.clientTools.values()).map((t) => ({
-          name: t.name,
-          description: t.description,
-          inputSchema: t.inputSchema
-            ? convertSchemaToJsonSchema(t.inputSchema)
-            : undefined,
-        }))
+        name: t.name,
+        description: t.description,
+        inputSchema: t.inputSchema
+          ? convertSchemaToJsonSchema(t.inputSchema)
+          : undefined,
+      }))
       : undefined
 
     this.connection.updateSession({

packages/typescript/ai-client/src/realtime-types.ts

@@ -8,6 +8,7 @@ import type {
   RealtimeSessionConfig,
   RealtimeStatus,
   RealtimeToken,
+  UsageInfo,
 } from '@tanstack/ai'
 
 // ============================================================================
@@ -25,11 +26,12 @@ export interface RealtimeAdapter {
   /**
    * Create a connection using the provided token
    * @param token - The ephemeral token from the server
-   * @param clientTools - Optional client-side tools to register with the provider
+   * @param config - Initial session configuration (voice, instructions, etc.)
    * @returns A connection instance
    */
   connect: (
     token: RealtimeToken,
+    config: RealtimeSessionConfig,
     clientTools?: ReadonlyArray<AnyClientTool>,
   ) => Promise<RealtimeConnection>
 }
@@ -148,6 +150,11 @@ export interface RealtimeClientOptions {
    */
   semanticEagerness?: 'low' | 'medium' | 'high'
 
+  /**
+   * Provider-specific options
+   */
+  providerOptions?: Record<string, unknown>
+
   // Callbacks
   onStatusChange?: (status: RealtimeStatus) => void
   onModeChange?: (mode: RealtimeMode) => void
@@ -156,6 +163,8 @@ export interface RealtimeClientOptions {
   onConnect?: () => void
   onDisconnect?: () => void
   onInterrupted?: () => void
+  onUsage?: (usage: UsageInfo) => void
+  onGoAway?: (timeLeft?: string) => void
 }
 
 // ============================================================================

packages/typescript/ai-elevenlabs/src/realtime/adapter.ts

@@ -1,4 +1,5 @@
 import { Conversation } from '@11labs/client'
+import { createRealtimeEventEmitter } from '@tanstack/ai'
 import type {
   AnyClientTool,
   AudioVisualization,
@@ -40,9 +41,10 @@ export function elevenlabsRealtime(
 
     async connect(
       token: RealtimeToken,
+      _config: RealtimeSessionConfig,
       clientToolDefs?: ReadonlyArray<AnyClientTool>,
     ): Promise<RealtimeConnection> {
-      return createElevenLabsConnection(token, options, clientToolDefs)
+      return createElevenLabsConnection(token, clientToolDefs)
     },
   }
 }
@@ -52,10 +54,9 @@ export function elevenlabsRealtime(
  */
 async function createElevenLabsConnection(
   token: RealtimeToken,
-  _options: ElevenLabsRealtimeOptions,
   clientToolDefs?: ReadonlyArray<AnyClientTool>,
 ): Promise<RealtimeConnection> {
-  const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()
+  const { emit, on: realtimeEventEmitterOn } = createRealtimeEventEmitter()
   let conversation: Awaited<
     ReturnType<typeof Conversation.startSession>
   > | null = null
@@ -65,19 +66,6 @@ async function createElevenLabsConnection(
   const emptyFrequencyData = new Uint8Array(128)
   const emptyTimeDomainData = new Uint8Array(128).fill(128)
 
-  // Helper to emit events
-  function emit<TEvent extends RealtimeEvent>(
-    event: TEvent,
-    payload: Parameters<RealtimeEventHandler<TEvent>>[0],
-  ) {
-    const handlers = eventHandlers.get(event)
-    if (handlers) {
-      for (const handler of handlers) {
-        handler(payload)
-      }
-    }
-  }
-
   function generateMessageId(): string {
     return `el-msg-${Date.now()}-${++messageIdCounter}`
   }
@@ -223,19 +211,7 @@ async function createElevenLabsConnection(
       emit('interrupted', {})
     },
 
-    on<TEvent extends RealtimeEvent>(
-      event: TEvent,
-      handler: RealtimeEventHandler<TEvent>,
-    ): () => void {
-      if (!eventHandlers.has(event)) {
-        eventHandlers.set(event, new Set())
-      }
-      eventHandlers.get(event)!.add(handler)
-
-      return () => {
-        eventHandlers.get(event)?.delete(handler)
-      }
-    },
+    on: realtimeEventEmitterOn,
 
     getAudioVisualization(): AudioVisualization {
       return {

packages/typescript/ai-gemini/package.json

@@ -40,14 +40,16 @@
     "adapter"
   ],
   "dependencies": {
-    "@google/genai": "^1.43.0"
+    "@google/genai": "^1.46.0"
   },
   "peerDependencies": {
-    "@tanstack/ai": "workspace:^"
+    "@tanstack/ai": "workspace:^",
+    "@tanstack/ai-client": "workspace:^"
   },
   "devDependencies": {
     "@tanstack/ai": "workspace:*",
+    "@tanstack/ai-client": "workspace:*",
     "@vitest/coverage-v8": "4.0.14",
-    "vite": "^7.2.7"
+    "vite": "^7.3.1"
   }
 }

packages/typescript/ai-gemini/src/index.ts

@@ -82,3 +82,16 @@ export type {
   GeminiDocumentMetadata,
   GeminiMessageMetadataByModality,
 } from './message-types'
+
+// ============================================================================
+// Realtime (Voice) Adapters
+// ============================================================================
+
+export { geminiRealtime, geminiRealtimeToken } from './realtime/index'
+
+export type {
+  GeminiRealtimeModel,
+  GeminiRealtimeOptions,
+  GeminiRealtimeTokenOptions,
+  GeminiRealtimeVoice,
+} from './realtime/index'