Realtime: send session.update event at connection/handoffs (#1112)

This is what tells the model about tools, tool choice, model etc etc.

Author: rm-openai

examples/realtime/demo.py

@@ -58,55 +58,45 @@ async def run(self) -> None:
             self.session = session
             self.ui.set_is_connected(True)
             async for event in session:
-                await self.on_event(event)
+                await self._on_event(event)
+            print("done")
 
         # Wait for UI task to complete when session ends
         await ui_task
 
     async def on_audio_recorded(self, audio_bytes: bytes) -> None:
-        """Called when audio is recorded by the UI."""
-        try:
-            # Send the audio to the session
-            assert self.session is not None
-            await self.session.send_audio(audio_bytes)
-        except Exception as e:
-            self.ui.log_message(f"Error sending audio: {e}")
-
-    async def on_event(self, event: RealtimeSessionEvent) -> None:
-        # Display event in the UI
-        try:
-            if event.type == "agent_start":
-                self.ui.add_transcript(f"Agent started: {event.agent.name}")
-            elif event.type == "agent_end":
-                self.ui.add_transcript(f"Agent ended: {event.agent.name}")
-            elif event.type == "handoff":
-                self.ui.add_transcript(
-                    f"Handoff from {event.from_agent.name} to {event.to_agent.name}"
-                )
-            elif event.type == "tool_start":
-                self.ui.add_transcript(f"Tool started: {event.tool.name}")
-            elif event.type == "tool_end":
-                self.ui.add_transcript(f"Tool ended: {event.tool.name}; output: {event.output}")
-            elif event.type == "audio_end":
-                self.ui.add_transcript("Audio ended")
-            elif event.type == "audio":
-                np_audio = np.frombuffer(event.audio.data, dtype=np.int16)
-                self.ui.play_audio(np_audio)
-            elif event.type == "audio_interrupted":
-                self.ui.add_transcript("Audio interrupted")
-            elif event.type == "error":
-                self.ui.add_transcript(f"Error: {event.error}")
-            elif event.type == "history_updated":
-                pass
-            elif event.type == "history_added":
-                pass
-            elif event.type == "raw_model_event":
-                self.ui.log_message(f"Raw model event: {event.data}")
-            else:
-                self.ui.log_message(f"Unknown event type: {event.type}")
-        except Exception as e:
-            # This can happen if the UI has already exited
-            self.ui.log_message(f"Event handling error: {str(e)}")
+        # Send the audio to the session
+        assert self.session is not None
+        await self.session.send_audio(audio_bytes)
+
+    async def _on_event(self, event: RealtimeSessionEvent) -> None:
+        if event.type == "agent_start":
+            self.ui.add_transcript(f"Agent started: {event.agent.name}")
+        elif event.type == "agent_end":
+            self.ui.add_transcript(f"Agent ended: {event.agent.name}")
+        elif event.type == "handoff":
+            self.ui.add_transcript(f"Handoff from {event.from_agent.name} to {event.to_agent.name}")
+        elif event.type == "tool_start":
+            self.ui.add_transcript(f"Tool started: {event.tool.name}")
+        elif event.type == "tool_end":
+            self.ui.add_transcript(f"Tool ended: {event.tool.name}; output: {event.output}")
+        elif event.type == "audio_end":
+            self.ui.add_transcript("Audio ended")
+        elif event.type == "audio":
+            np_audio = np.frombuffer(event.audio.data, dtype=np.int16)
+            self.ui.play_audio(np_audio)
+        elif event.type == "audio_interrupted":
+            self.ui.add_transcript("Audio interrupted")
+        elif event.type == "error":
+            self.ui.add_transcript(f"Error: {event.error}")
+        elif event.type == "history_updated":
+            pass
+        elif event.type == "history_added":
+            pass
+        elif event.type == "raw_model_event":
+            self.ui.log_message(f"Raw model event: {event.data}")
+        else:
+            self.ui.log_message(f"Unknown event type: {event.type}")
 
 
 if __name__ == "__main__":

examples/realtime/ui.py

@@ -239,10 +239,7 @@ async def capture_audio(self) -> None:
 
                 # Call audio callback if set
                 if self.audio_callback:
-                    try:
-                        await self.audio_callback(audio_bytes)
-                    except Exception as e:
-                        self.log_message(f"Audio callback error: {e}")
+                    await self.audio_callback(audio_bytes)
 
                 # Yield control back to event loop
                 await asyncio.sleep(0)

src/agents/realtime/model_inputs.py

@@ -5,6 +5,7 @@
 
 from typing_extensions import NotRequired, TypeAlias, TypedDict
 
+from .config import RealtimeSessionModelSettings
 from .model_events import RealtimeModelToolCallEvent
 
 
@@ -81,10 +82,19 @@ class RealtimeModelSendInterrupt:
     """Send an interrupt to the model."""
 
 
+@dataclass
+class RealtimeModelSendSessionUpdate:
+    """Send a session update to the model."""
+
+    session_settings: RealtimeSessionModelSettings
+    """The updated session settings to send."""
+
+
 RealtimeModelSendEvent: TypeAlias = Union[
     RealtimeModelSendRawMessage,
     RealtimeModelSendUserInput,
     RealtimeModelSendAudio,
     RealtimeModelSendToolOutput,
     RealtimeModelSendInterrupt,
+    RealtimeModelSendSessionUpdate,
 ]