feat: tool abstraction (#356)

* feat: version updateg * feat: update build image * fix: upgrade agora_rtc and azure_tts to fix build * fix: fix build failure * fix: fix property.jso * fix: fix graph names * fix: remove app: localhost * fix: update frontend image * feat: turn openai python extension to async extension * fix: incompatible version * fix: update agora_rtc and azure_tts to fix logs * property.json update * fix: fix build failure * fix: fix bingsearch manifest * feat: tool abstraction * feat: support vision tool * finalize * feat: support tool update * fix: fix issues - use member variable instead of class - set system package variable in bin/start - remove token in property.json * fix: fix on comments - added comments * rename variables --------- Co-authored-by: Jay Zhang <[email protected]>
TEN-framework · Oct 29, 2024 · c21ef0b · c21ef0b
1 parent b2e2a04
commit c21ef0b
Show file tree

Hide file tree

Showing 23 changed files with 870 additions and 286 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,6 +1,7 @@
 {
   "python.analysis.extraPaths": [
     "./agents/ten_packages/system/ten_runtime_python/interface",
+    "./agents/ten_packages/system/ten_ai_base/interface",
   ],
   "C_Cpp.intelliSenseEngine": "disabled",
   "editor.formatOnSave": true,

diff --git a/agents/bin/start b/agents/bin/start
@@ -4,6 +4,7 @@ set -e
 
 cd "$(dirname "${BASH_SOURCE[0]}")/.."
 
+export PYTHONPATH=/app/agents/ten_packages/system/ten_ai_base/interface:$PYTHONPATH
 export LD_LIBRARY_PATH=$(pwd)/ten_packages/system/agora_rtc_sdk/lib:$(pwd)/ten_packages/system/azure_speech_sdk/lib
 
 exec bin/worker "$@"
diff --git a/agents/property.json b/agents/property.json
@@ -1640,7 +1640,7 @@
             },
             {
                 "name": "camera_va_openai_azure",
-                "auto_start": false,
+                "auto_start": true,
                 "nodes": [
                     {
                         "type": "extension",
@@ -1649,7 +1649,7 @@
                         "name": "agora_rtc",
                         "property": {
                             "app_id": "${env:AGORA_APP_ID}",
-                            "token": "<agora_token>",
+                            "token": "",
                             "channel": "ten_agent_test",
                             "stream_id": 1234,
                             "remote_stream_id": 123,
@@ -1681,7 +1681,7 @@
                             "base_url": "${env:OPENAI_API_BASE}",
                             "api_key": "${env:OPENAI_API_KEY}",
                             "frequency_penalty": 0.9,
-                            "model": "${env:OPENAI_MODEL}",
+                            "model": "gpt-4o",
                             "max_tokens": 512,
                             "prompt": "",
                             "proxy_url": "${env:OPENAI_PROXY_URL}",
@@ -1691,6 +1691,12 @@
                             "enable_tools": true
                         }
                     },
+                    {
+                        "type": "extension",
+                        "extension_group": "chatgpt",
+                        "addon": "vision_tool_python",
+                        "name": "vision_tool"
+                    },
                     {
                         "type": "extension",
                         "extension_group": "tts",
@@ -1738,7 +1744,7 @@
                                 "dest": [
                                     {
                                         "extension_group": "chatgpt",
-                                        "extension": "openai_chatgpt"
+                                        "extension": "vision_tool"
                                     }
                                 ]
                             }
@@ -1791,6 +1797,30 @@
                                         "extension": "azure_tts"
                                     }
                                 ]
+                            },
+                            {
+                                "name": "tool_call",
+                                "dest": [
+                                    {
+                                        "extension_group": "chatgpt",
+                                        "extension": "vision_tool"
+                                    }
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        "extension_group": "chatgpt",
+                        "extension": "vision_tool",
+                        "cmd": [
+                            {
+                                "name": "tool_register",
+                                "dest": [
+                                    {
+                                        "extension_group": "chatgpt",
+                                        "extension": "openai_chatgpt"
+                                    }
+                                ]
                             }
                         ]
                     },
@@ -2408,7 +2438,7 @@
             },
             {
                 "name": "va_openai_v2v",
-                "auto_start": true,
+                "auto_start": false,
                 "nodes": [
                     {
                         "type": "extension",

diff --git a/agents/scripts/install_deps_and_build.sh b/agents/scripts/install_deps_and_build.sh
@@ -65,6 +65,15 @@ install_python_requirements() {
     done
   fi
 
+  # traverse the ten_packages/system directory to find the requirements.txt
+  if [[ -d "ten_packages/system" ]]; then
+    for extension in ten_packages/system/*; do
+      if [[ -f "$extension/requirements.txt" ]]; then
+        pip install -r $extension/requirements.txt
+      fi
+    done
+  fi
+
   # pre-import llama-index as it cloud download additional resources during the first import
   echo "pre-import python modules..."
   python3.10 -c "import llama_index.core;"