wip: changes with the latest openai package

Author: seratch

examples/realtime/app/server.py

@@ -101,7 +101,11 @@ async def _serialize_event(self, event: RealtimeSessionEvent) -> dict[str, Any]:
         elif event.type == "history_updated":
             base_event["history"] = [item.model_dump(mode="json") for item in event.history]
         elif event.type == "history_added":
-            pass
+            # Provide the added item so the UI can render incrementally.
+            try:
+                base_event["item"] = event.item.model_dump(mode="json")
+            except Exception:
+                base_event["item"] = None
         elif event.type == "guardrail_tripped":
             base_event["guardrail_results"] = [
                 {"name": result.guardrail.name} for result in event.guardrail_results

examples/realtime/app/static/app.js

@@ -210,6 +210,12 @@ class RealtimeDemo {
             case 'history_updated':
                 this.updateMessagesFromHistory(event.history);
                 break;
+            case 'history_added':
+                // Append just the new item without clearing the thread.
+                if (event.item) {
+                    this.addMessageFromItem(event.item);
+                }
+                break;
         }
     }
 
@@ -235,13 +241,7 @@ class RealtimeDemo {
                         // Extract text from content array
                         item.content.forEach(contentPart => {
                             console.log('Content part:', contentPart);
-                            if (contentPart.type === 'text' && contentPart.text) {
-                                content += contentPart.text;
-                            } else if (contentPart.type === 'input_text' && contentPart.text) {
-                                content += contentPart.text;
-                            } else if (contentPart.type === 'input_audio' && contentPart.transcript) {
-                                content += contentPart.transcript;
-                            } else if (contentPart.type === 'audio' && contentPart.transcript) {
+                            if (contentPart && contentPart.transcript) {
                                 content += contentPart.transcript;
                             }
                         });
@@ -263,6 +263,35 @@ class RealtimeDemo {
 
         this.scrollToBottom();
     }
+
+    addMessageFromItem(item) {
+        try {
+            if (!item || item.type !== 'message') return;
+            const role = item.role;
+            let content = '';
+
+            if (Array.isArray(item.content)) {
+                for (const contentPart of item.content) {
+                    if (!contentPart || typeof contentPart !== 'object') continue;
+                    if (contentPart.type === 'text' && contentPart.text) {
+                        content += contentPart.text;
+                    } else if (contentPart.type === 'input_text' && contentPart.text) {
+                        content += contentPart.text;
+                    } else if (contentPart.type === 'input_audio' && contentPart.transcript) {
+                        content += contentPart.transcript;
+                    } else if (contentPart.type === 'audio' && contentPart.transcript) {
+                        content += contentPart.transcript;
+                    }
+                }
+            }
+
+            if (content && content.trim()) {
+                this.addMessage(role, content.trim());
+            }
+        } catch (e) {
+            console.error('Failed to add message from item:', e, item);
+        }
+    }
 
     addMessage(type, content) {
         const messageDiv = document.createElement('div');
@@ -464,4 +493,4 @@ class RealtimeDemo {
 // Initialize the demo when the page loads
 document.addEventListener('DOMContentLoaded', () => {
     new RealtimeDemo();
-});
+});

examples/realtime/app/static/favicon.ico

@@ -280,7 +280,7 @@ async def _on_event(self, event: RealtimeSessionEvent) -> None:
             elif event.type == "history_added":
                 pass  # Skip these frequent events
             elif event.type == "raw_model_event":
-                print(f"Raw model event: {_truncate_str(str(event.data), 50)}")
+                print(f"Raw model event: {_truncate_str(str(event.data), 200)}")
             else:
                 print(f"Unknown event type: {event.type}")
         except Exception as e:

examples/realtime/cli/demo.py

@@ -4,6 +4,6 @@
 
 
 def calculate_audio_length_ms(format: RealtimeAudioFormat | None, audio_bytes: bytes) -> float:
-    if format and format.startswith("g711"):
+    if format and isinstance(format, str) and format.startswith("g711"):
         return (len(audio_bytes) / 8000) * 1000
     return (len(audio_bytes) / 24 / 2) * 1000

src/agents/realtime/_util.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from openai.types.realtime.realtime_audio_formats import (
+    AudioPCM,
+    AudioPCMA,
+    AudioPCMU,
+    RealtimeAudioFormats,
+)
+
+from ..logger import logger
+
+type LegacyRealtimeAudioFormats = Literal["pcm16", "g711_ulaw", "g711_alaw"]
+
+
+def to_realtime_audio_format(
+    input_audio_format: LegacyRealtimeAudioFormats | RealtimeAudioFormats | None,
+) -> RealtimeAudioFormats | None:
+    format: RealtimeAudioFormats | None = None
+    if input_audio_format is not None:
+        if isinstance(input_audio_format, str):
+            if input_audio_format in ["pcm16", "audio/pcm", "pcm"]:
+                format = AudioPCM(type="audio/pcm", rate=24000)
+            elif input_audio_format in ["g711_ulaw", "audio/pcmu", "pcmu"]:
+                format = AudioPCMU(type="audio/pcmu")
+            elif input_audio_format in ["g711_alaw", "audio/pcma", "pcma"]:
+                format = AudioPCMA(type="audio/pcma")
+            else:
+                logger.debug(f"Unknown input_audio_format: {input_audio_format}")
+        else:
+            format = input_audio_format
+    return format

src/agents/realtime/audio_formats.py

@@ -6,6 +6,9 @@
     Union,
 )
 
+from openai.types.realtime.realtime_audio_formats import (
+    RealtimeAudioFormats as OpenAIRealtimeAudioFormats,
+)
 from typing_extensions import NotRequired, TypeAlias, TypedDict
 
 from agents.prompts import Prompt
@@ -107,10 +110,10 @@ class RealtimeSessionModelSettings(TypedDict):
     speed: NotRequired[float]
     """The speed of the model's responses."""
 
-    input_audio_format: NotRequired[RealtimeAudioFormat]
+    input_audio_format: NotRequired[RealtimeAudioFormat | OpenAIRealtimeAudioFormats]
     """The format for input audio streams."""
 
-    output_audio_format: NotRequired[RealtimeAudioFormat]
+    output_audio_format: NotRequired[RealtimeAudioFormat | OpenAIRealtimeAudioFormats]
     """The format for output audio streams."""
 
     input_audio_transcription: NotRequired[RealtimeInputAudioTranscriptionConfig]