add deepgram into vision graph

TomasBack2Future · Nov 13, 2024 · 3384a84 · 3384a84
1 parent da5eac8
commit 3384a84
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 22 deletions.
diff --git a/agents/property.json b/agents/property.json
@@ -4294,15 +4294,21 @@
               "subscribe_video": true,
               "publish_audio": true,
               "publish_data": true,
-              "enable_agora_asr": true,
-              "agora_asr_vendor_name": "microsoft",
-              "agora_asr_language": "en-US",
-              "agora_asr_vendor_key": "${env:AZURE_STT_KEY}",
-              "agora_asr_vendor_region": "${env:AZURE_STT_REGION}",
-              "agora_asr_session_control_file_path": "session_control.conf",
               "subscribe_video_pix_fmt": 4
             }
           },
+          {
+            "type": "extension",
+            "extension_group": "asr",
+            "addon": "deepgram_asr_python",
+            "name": "deepgram_asr",
+            "property": {
+              "api_key": "${env:DEEPGRAM_API_KEY}",
+              "language": "en-US",
+              "model": "nova-2",
+              "sample_rate": "16000"
+            }
+          },
           {
             "type": "extension",
             "extension_group": "default",
@@ -4346,21 +4352,13 @@
           {
             "extension_group": "default",
             "extension": "agora_rtc",
-            "data": [
+            "audio_frame": [
               {
-                "name": "text_data",
+                "name": "pcm_frame",
                 "dest": [
                   {
-                    "extension_group": "default",
-                    "extension": "interrupt_detector"
-                  },
-                  {
-                    "extension_group": "llm",
-                    "extension": "vision_python_async"
-                  },
-                  {
-                    "extension_group": "transcriber",
-                    "extension": "message_collector"
+                    "extension_group": "asr",
+                    "extension": "deepgram_asr"
                   }
                 ]
               }
@@ -4397,6 +4395,29 @@
               }
             ]
           },
+          {
+            "extension_group": "asr",
+            "extension": "deepgram_asr",
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension_group": "default",
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension_group": "llm",
+                    "extension": "vision_python_async"
+                  },
+                  {
+                    "extension_group": "transcriber",
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
           {
             "extension_group": "llm",
             "extension": "vision_python_async",

diff --git a/agents/ten_packages/extension/deepgram_asr_python/deepgram_wrapper.py b/agents/ten_packages/extension/deepgram_asr_python/deepgram_wrapper.py
@@ -89,8 +89,7 @@ async def on_error(self, error, **kwargs):
     async def send_frame(self) -> None:
         while not self.stopped:
             try:
-                pcm_frame = await asyncio.wait_for(self.queue.get(), timeout=10.0)
-
+                pcm_frame = await self.queue.get()
                 if pcm_frame is None:
                     logger.warning("send_frame: exit due to None value got.")
                     return
@@ -103,8 +102,6 @@ async def send_frame(self) -> None:
                 self.stream_id = pcm_frame.get_property_int('stream_id')
                 await self.deepgram_client.send(frame_buf)
                 self.queue.task_done()
-            except asyncio.TimeoutError as e:
-                logger.exception(f"error in send_frame: {e}")
             except IOError as e:
                 logger.exception(f"error in send_frame: {e}")
             except Exception as e: