feat: add minimax v2v (#368)

* feat: init minimax v2v * feat: async http client * feat: interruptable * feat: api * fix: data stream chunking for transcript display * docs: readme
TEN-framework · Oct 29, 2024 · 76e8b72 · 76e8b72
1 parent e875cbd
commit 76e8b72
Show file tree

Hide file tree

Showing 15 changed files with 1,071 additions and 118 deletions.
diff --git a/agents/.gitignore b/agents/.gitignore
@@ -2,6 +2,7 @@
 ten_packages/extension_group/
 ten_packages/extension/agora_rtc
 ten_packages/extension/azure_tts
+ten_packages/extension/agora_sess_ctrl
 ten_packages/extension/py_init_extension_cpp
 ten_packages/system
 .ten

diff --git a/agents/manifest-lock.json b/agents/manifest-lock.json
@@ -67,6 +67,24 @@
         }
       ]
     },
+    {
+      "type": "extension",
+      "name": "agora_sess_ctrl",
+      "version": "0.2.0",
+      "hash": "52f26dee2fb8fbd22d55ecc6bb197176a51f4a3bd2268788f75582f68cf1270b",
+      "dependencies": [
+        {
+          "type": "system",
+          "name": "ten_runtime"
+        }
+      ],
+      "supports": [
+        {
+          "os": "linux",
+          "arch": "x64"
+        }
+      ]
+    },
     {
       "type": "system",
       "name": "azure_speech_sdk",

diff --git a/agents/manifest.json b/agents/manifest.json
@@ -18,6 +18,11 @@
       "name": "agora_rtc",
       "version": "=0.8.0-rc2"
     },
+    {
+      "type": "extension",
+      "name": "agora_sess_ctrl",
+      "version": "0.2.0"
+    },
     {
       "type": "system",
       "name": "azure_speech_sdk",

diff --git a/agents/property.json b/agents/property.json
@@ -3474,7 +3474,160 @@
                         ]
                     }
                 ]
+            },
+            {
+                "name": "va_minimax_v2v",
+                "auto_start": false,
+                "nodes": [
+                    {
+                        "type": "extension",
+                        "extension_group": "rtc",
+                        "addon": "agora_rtc",
+                        "name": "agora_rtc",
+                        "property": {
+                            "app_id": "${env:AGORA_APP_ID}",
+                            "token": "",
+                            "channel": "ten_agent_test",
+                            "stream_id": 1234,
+                            "remote_stream_id": 123,
+                            "subscribe_audio": true,
+                            "publish_audio": true,
+                            "publish_data": true
+                        }
+                    },
+                    {
+                        "type": "extension",
+                        "extension_group": "agora_sess_ctrl",
+                        "addon": "agora_sess_ctrl",
+                        "name": "agora_sess_ctrl",
+                        "property": {
+                            "wait_for_eos": true
+                        }
+                    },
+                    {
+                        "type": "extension",
+                        "extension_group": "llm",
+                        "addon": "minimax_v2v_python",
+                        "name": "minimax_v2v_python",
+                        "property": {
+                            "in_sample_rate": 16000,
+                            "token": "${env:MINIMAX_TOKEN}"
+                        }
+                    },
+                    {
+                        "type": "extension",
+                        "extension_group": "message_collector",
+                        "addon": "message_collector",
+                        "name": "message_collector"
+                    }
+                ],
+                "connections": [
+                    {
+                        "extension_group": "rtc",
+                        "extension": "agora_rtc",
+                        "audio_frame": [
+                            {
+                                "name": "pcm_frame",
+                                "dest": [
+                                    {
+                                        "extension_group": "agora_sess_ctrl",
+                                        "extension": "agora_sess_ctrl"
+                                    }
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        "extension_group": "agora_sess_ctrl",
+                        "extension": "agora_sess_ctrl",
+                        "audio_frame": [
+                            {
+                                "name": "pcm_frame",
+                                "dest": [
+                                    {
+                                        "extension_group": "llm",
+                                        "extension": "minimax_v2v_python"
+                                    }
+                                ]
+                            }
+                        ],
+                        "cmd": [
+                            {
+                                "name": "start_of_sentence",
+                                "dest": [
+                                    {
+                                        "extension_group": "llm",
+                                        "extension": "minimax_v2v_python",
+                                        "msg_conversion": {
+                                            "type": "per_property",
+                                            "keep_original": true,
+                                            "rules": [
+                                                {
+                                                    "path": "_ten.name",
+                                                    "conversion_mode": "fixed_value",
+                                                    "value": "flush"
+                                                }
+                                            ]
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        "extension_group": "llm",
+                        "extension": "minimax_v2v_python",
+                        "data": [
+                            {
+                                "name": "text_data",
+                                "dest": [
+                                    {
+                                        "extension_group": "message_collector",
+                                        "extension": "message_collector"
+                                    }
+                                ]
+                            }
+                        ],
+                        "audio_frame": [
+                            {
+                                "name": "pcm_frame",
+                                "dest": [
+                                    {
+                                        "extension_group": "rtc",
+                                        "extension": "agora_rtc"
+                                    }
+                                ]
+                            }
+                        ],
+                        "cmd": [
+                            {
+                                "name": "flush",
+                                "dest": [
+                                    {
+                                        "extension_group": "rtc",
+                                        "extension": "agora_rtc"
+                                    }
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        "extension_group": "message_collector",
+                        "extension": "message_collector",
+                        "data": [
+                            {
+                                "name": "data",
+                                "dest": [
+                                    {
+                                        "extension_group": "rtc",
+                                        "extension": "agora_rtc"
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                ]
             }
         ]
     }
-}
+}