upload save record wav file

Author: lloydzhou

app/components/realtime-chat/realtime-chat.tsx

@@ -127,7 +127,6 @@ export function RealtimeChat({
 
   const handleResponse = async (response: RTResponse) => {
     for await (const item of response) {
-      console.log("handleResponse", item);
       if (item.type === "message" && item.role === "assistant") {
         const botMessage = createMessage({
           role: item.role,
@@ -156,12 +155,16 @@ export function RealtimeChat({
             };
             await Promise.all([textTask(), audioTask()]);
           }
+          // update message.content
+          chatStore.updateTargetSession((session) => {
+            session.messages = session.messages.concat();
+          });
         }
         // upload audio get audio_url
         const blob = audioHandlerRef.current?.savePlayFile();
         uploadImage(blob).then((audio_url) => {
           botMessage.audio_url = audio_url;
-          botMessage.date = new Date().toLocaleString();
+          // botMessage.date = new Date().toLocaleString();
           // update text and audio_url
           chatStore.updateTargetSession((session) => {
             session.messages = session.messages.concat();
@@ -174,16 +177,28 @@ export function RealtimeChat({
   const handleInputAudio = async (item: RTInputAudioItem) => {
     audioHandlerRef.current?.stopStreamingPlayback();
     await item.waitForCompletion();
-    const { audioStartMillis, audioEndMillis } = item;
-    // TODO, save input audio_url, and update session
-    console.log("handleInputAudio", item, audioStartMillis, audioEndMillis);
-    const userMessage = createMessage({
-      role: "user",
-      content: item.transcription,
-    });
-    chatStore.updateTargetSession(session, (session) => {
-      session.messages = session.messages.concat([userMessage]);
-    });
+    if (item.transcription) {
+      const userMessage = createMessage({
+        role: "user",
+        content: item.transcription,
+      });
+      chatStore.updateTargetSession(session, (session) => {
+        session.messages = session.messages.concat([userMessage]);
+      });
+      // save input audio_url, and update session
+      const { audioStartMillis, audioEndMillis } = item;
+      // upload audio get audio_url
+      const blob = audioHandlerRef.current?.saveRecordFile(
+        audioStartMillis,
+        audioEndMillis,
+      );
+      uploadImage(blob).then((audio_url) => {
+        userMessage.audio_url = audio_url;
+        chatStore.updateTargetSession((session) => {
+          session.messages = session.messages.concat();
+        });
+      });
+    }
   };
 
   const toggleRecording = async () => {

app/lib/audio.ts

@@ -3,6 +3,7 @@ export class AudioHandler {
   private workletNode: AudioWorkletNode | null = null;
   private stream: MediaStream | null = null;
   private source: MediaStreamAudioSourceNode | null = null;
+  private recordBuffer: Int16Array[] = [];
   private readonly sampleRate = 24000;
 
   private nextPlayTime: number = 0;
@@ -52,6 +53,8 @@ export class AudioHandler {
 
           const uint8Data = new Uint8Array(int16Data.buffer);
           onChunk(uint8Data);
+          // save recordBuffer
+          this.recordBuffer.push.apply(this.recordBuffer, int16Data);
         }
       };
 
@@ -154,7 +157,22 @@ export class AudioHandler {
   savePlayFile() {
     return this._saveData(new Int16Array(this.playBuffer));
   }
+  saveRecordFile(
+    audioStartMillis: number | undefined,
+    audioEndMillis: number | undefined,
+  ) {
+    const startIndex = audioStartMillis
+      ? Math.floor((audioStartMillis * this.sampleRate) / 1000)
+      : 0;
+    const endIndex = audioEndMillis
+      ? Math.floor((audioEndMillis * this.sampleRate) / 1000)
+      : this.recordBuffer.length;
+    return this._saveData(
+      new Int16Array(this.recordBuffer.slice(startIndex, endIndex)),
+    );
+  }
   async close() {
+    this.recordBuffer = [];
     this.workletNode?.disconnect();
     this.source?.disconnect();
     this.stream?.getTracks().forEach((track) => track.stop());

app/utils/chat.ts

@@ -138,7 +138,7 @@ export function uploadImage(file: Blob): Promise<string> {
   })
     .then((res) => res.json())
     .then((res) => {
-      console.log("res", res);
+      // console.log("res", res);
       if (res?.code == 0 && res?.data) {
         return res?.data;
       }