diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index aed0cfcd..8e043c6c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ // README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile { "name": "ten_agent_dev", - "image": "ghcr.io/ten-framework/ten_agent_build:0.1.0", + "image": "ghcr.io/ten-framework/ten_agent_build:0.2.1", "customizations": { "vscode": { "extensions": [ diff --git a/.env.example b/.env.example index 4713a75f..7608d59a 100644 --- a/.env.example +++ b/.env.example @@ -96,3 +96,12 @@ OPENAI_PROXY_URL= # Extension: qwen_llm # Qwen API key QWEN_API_KEY= + + +# Extension: weatherapi_tool_python +# Weathers API key +WEATHERAPI_API_KEY= + +# Extension: bingsearch_tool_python +# Bing search API key +BING_API_KEY= \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 97adf23a..fd2aecf4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ten-framework/ten_agent_build:0.1.0 AS builder +FROM ghcr.io/ten-framework/ten_agent_build:0.2.1 AS builder ARG SESSION_CONTROL_CONF=session_control.conf diff --git a/agents/main.go b/agents/main.go index b42101f5..0923cd50 100644 --- a/agents/main.go +++ b/agents/main.go @@ -25,19 +25,19 @@ type defaultApp struct { cfg *appConfig } -func (p *defaultApp) OnInit( +func (p *defaultApp) OnConfigure( tenEnv ten.TenEnv, ) { // Using the default property.json if not specified. if len(p.cfg.PropertyFilePath) > 0 { if b, err := os.ReadFile(p.cfg.PropertyFilePath); err != nil { log.Fatalf("Failed to read property file %s, err %v\n", p.cfg.PropertyFilePath, err) - }else{ + } else { tenEnv.InitPropertyFromJSONBytes(b) } } - tenEnv.OnInitDone() + tenEnv.OnConfigureDone() } func startAppBlocking(cfg *appConfig) { diff --git a/agents/manifest-lock.json b/agents/manifest-lock.json index 67fe30ef..1398fb18 100644 --- a/agents/manifest-lock.json +++ b/agents/manifest-lock.json @@ -4,8 +4,8 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.2.0", - "hash": "8b582de5dfaa38983104143fbb6c530b3aeb463ad9e9ef51f2c72fba9862b8cc", + "version": "0.3.1", + "hash": "304075ff197ad241fcb2fe066bb0096e5fc7ecc3ba163e463b38e950f31586d2", "dependencies": [ { "type": "system", @@ -22,8 +22,8 @@ { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.2.0", - "hash": "e1858dfd83d18a69901cefb2edfd52e57ee326a3d306e799ff1d661f3195bb6b", + "version": "0.3.1", + "hash": "5b232d51213657fc2977be6d7de379a04bf4dcb955f555ac8a55baaab4c20b30", "dependencies": [ { "type": "system", @@ -41,29 +41,11 @@ } ] }, - { - "type": "extension_group", - "name": "default_extension_group", - "version": "0.2.0", - "hash": "117ed3e747654fc1282129a160eaecc2cd16548e70aa22128efee21f10e185c8", - "dependencies": [ - { - "type": "system", - "name": "ten_runtime" - } - ], - "supports": [ - { - "os": "linux", - "arch": "x64" - } - ] - }, { "type": "extension", "name": "agora_rtc", - "version": "0.7.0-rc2", - "hash": "89d7af8f84d06afbd79901e0057182280f3f430227ad6fae98ec154067ffa82c", + "version": "0.8.0-rc2", + "hash": "7b43c050cbec4f14e01df2e08f4ad9a99ef7ac37879affbff07693136a19cd2d", "dependencies": [ { "type": "system", @@ -100,8 +82,8 @@ { "type": "extension", "name": "azure_tts", - "version": "0.4.2", - "hash": "f1f3e58bd67ff45d537703690fe15f37f37b1e5754c29c483e74f15ee8fde7c8", + "version": "0.5.1", + "hash": "3876cd46a618658d980a71704e475fc7e3019d85a8ce6012653a7b12ff006a5c", "dependencies": [ { "type": "system", @@ -117,8 +99,8 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0", - "hash": "7effdb036d5bf91894060a9230775ff8ec2598f202b8238578f99a14dbf11632", + "version": "0.3.1", + "hash": "b39bd74b9ceedfcd356596890c62c239c2dce8d3d83bbca2dcff9912a1b21a88", "supports": [ { "os": "linux", @@ -129,8 +111,8 @@ { "type": "system", "name": "agora_rtc_sdk", - "version": "4.1.36+build331418", - "hash": "74115dd35822dc3b09fbadb577b31146af704d60435ca401c0b305cce80b2ba4", + "version": "4.1.38+build376742", + "hash": "c57acf53d90fbee04d06aa3fd48b0425dd4cca1bee62666c5c01699044b43288", "supports": [ { "os": "linux", @@ -148,8 +130,8 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2.0", - "hash": "b44d3767f364583f8bb8e8995f6f7f49e3af27b3c9a8ddf62fa319f1fc39910e", + "version": "0.3.1", + "hash": "949915b3385da478fe7f30c583aa6adb7b33f07f2278e063d5d733d77e4f0cec", "dependencies": [ { "type": "system", diff --git a/agents/manifest.json b/agents/manifest.json index a2488fd2..f92fe394 100644 --- a/agents/manifest.json +++ b/agents/manifest.json @@ -6,22 +6,17 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.2" + "version": "0.3" }, { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.2" - }, - { - "type": "extension_group", - "name": "default_extension_group", - "version": "0.2" + "version": "0.3" }, { "type": "extension", "name": "agora_rtc", - "version": "=0.7.0-rc2" + "version": "=0.8.0-rc2" }, { "type": "system", @@ -31,7 +26,7 @@ { "type": "extension", "name": "azure_tts", - "version": "0.4.2" + "version": "=0.5.1" } ] } \ No newline at end of file diff --git a/agents/property.json b/agents/property.json index 5b6f46a9..9f2a2867 100644 --- a/agents/property.json +++ b/agents/property.json @@ -3,26 +3,22 @@ "log_level": 3, "predefined_graphs": [ { - "name": "va.openai.azure.fashionai", - "auto_start": true, + "name": "va_openai_azure_fashionai", + "auto_start": false, "connections": [ { - "app": "localhost", "data": [ { "dest": [ { - "app": "localhost", "extension": "interrupt_detector", "extension_group": "default" }, { - "app": "localhost", "extension": "openai_chatgpt", "extension_group": "chatgpt" }, { - "app": "localhost", "extension": "message_collector", "extension_group": "transcriber" } @@ -34,17 +30,14 @@ "extension_group": "default" }, { - "app": "localhost", "cmd": [ { "dest": [ { - "app": "localhost", "extension": "azure_tts", "extension_group": "tts" }, { - "app": "localhost", "extension": "fashionai", "extension_group": "default" } @@ -56,12 +49,10 @@ { "dest": [ { - "app": "localhost", "extension": "message_collector", "extension_group": "transcriber" }, { - "app": "localhost", "extension": "fashionai", "extension_group": "default" } @@ -73,12 +64,10 @@ "extension_group": "chatgpt" }, { - "app": "localhost", "audio_frame": [ { "dest": [ { - "app": "localhost", "extension": "agora_rtc", "extension_group": "default" } @@ -90,7 +79,6 @@ { "dest": [ { - "app": "localhost", "extension": "agora_rtc", "extension_group": "default" } @@ -102,12 +90,10 @@ "extension_group": "tts" }, { - "app": "localhost", "data": [ { "dest": [ { - "app": "localhost", "extension": "agora_rtc", "extension_group": "default" } @@ -119,12 +105,10 @@ "extension_group": "transcriber" }, { - "app": "localhost", "cmd": [ { "dest": [ { - "app": "localhost", "extension": "openai_chatgpt", "extension_group": "chatgpt" } @@ -139,7 +123,6 @@ "nodes": [ { "addon": "agora_rtc", - "app": "localhost", "extension_group": "default", "name": "agora_rtc", "property": { @@ -162,14 +145,12 @@ }, { "addon": "interrupt_detector", - "app": "localhost", "extension_group": "default", "name": "interrupt_detector", "type": "extension" }, { "addon": "openai_chatgpt", - "app": "localhost", "extension_group": "chatgpt", "name": "openai_chatgpt", "property": { @@ -187,7 +168,6 @@ }, { "addon": "azure_tts", - "app": "localhost", "extension_group": "tts", "name": "azure_tts", "property": { @@ -199,14 +179,12 @@ }, { "addon": "message_collector", - "app": "localhost", "extension_group": "transcriber", "name": "message_collector", "type": "extension" }, { "addon": "fashionai", - "app": "localhost", "extension_group": "default", "name": "fashionai", "property": { @@ -221,7 +199,7 @@ ] }, { - "name": "va.openai.azure", + "name": "va_openai_azure", "auto_start": false, "nodes": [ { @@ -285,26 +263,6 @@ "extension_group": "transcriber", "addon": "message_collector", "name": "message_collector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "chatgpt" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "transcriber" } ], "connections": [ @@ -420,7 +378,7 @@ ] }, { - "name": "va.openai.11labs", + "name": "va_openai_11labs", "auto_start": false, "nodes": [ { @@ -490,26 +448,6 @@ "extension_group": "transcriber", "addon": "message_collector", "name": "message_collector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "chatgpt" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "transcriber" } ], "connections": [ @@ -625,7 +563,7 @@ ] }, { - "name": "va.openai.fish", + "name": "va_openai_fish", "auto_start": false, "nodes": [ { @@ -673,6 +611,12 @@ "max_memory_length": 10 } }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + }, { "type": "extension", "extension_group": "tts", @@ -685,32 +629,6 @@ "request_timeout_seconds": 30, "base_url": "https://api.fish.audio" } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "chatgpt" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "transcriber" } ], "connections": [ @@ -826,7 +744,7 @@ ] }, { - "name": "va.bedrock.azure", + "name": "va_bedrock_azure", "auto_start": false, "nodes": [ { @@ -883,21 +801,6 @@ "extension_group": "default", "addon": "interrupt_detector_python", "name": "interrupt_detector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "bedrock" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" } ], "connections": [ @@ -990,7 +893,7 @@ ] }, { - "name": "va.openai.cosy", + "name": "va_openai_cosy", "auto_start": false, "nodes": [ { @@ -1055,26 +958,6 @@ "extension_group": "chat_transcriber", "addon": "message_collector", "name": "message_collector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "chatgpt" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "chat_transcriber" } ], "connections": [ @@ -1210,7 +1093,7 @@ ] }, { - "name": "va.qwen.cosy", + "name": "va_qwen_cosy", "auto_start": false, "nodes": [ { @@ -1271,26 +1154,6 @@ "extension_group": "default", "addon": "interrupt_detector_python", "name": "interrupt_detector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "llm" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "chat_transcriber" } ], "connections": [ @@ -1413,7 +1276,7 @@ ] }, { - "name": "va.bedrock.polly", + "name": "va_bedrock_polly", "auto_start": false, "nodes": [ { @@ -1474,21 +1337,6 @@ "extension_group": "default", "addon": "interrupt_detector_python", "name": "interrupt_detector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "bedrock" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" } ], "connections": [ @@ -1581,7 +1429,7 @@ ] }, { - "name": "va.transcribe-bedrock.polly", + "name": "va_transcribe_bedrock_polly", "auto_start": false, "nodes": [ { @@ -1655,26 +1503,6 @@ "extension_group": "default", "addon": "interrupt_detector_python", "name": "interrupt_detector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "bedrock" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "asr" } ], "connections": [ @@ -1811,8 +1639,8 @@ ] }, { - "name": "camera.va.openai.azure", - "auto_start": true, + "name": "camera_va_openai_azure", + "auto_start": false, "nodes": [ { "type": "extension", @@ -1879,26 +1707,6 @@ "extension_group": "transcriber", "addon": "message_collector", "name": "message_collector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "chatgpt" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "transcriber" } ], "connections": [ @@ -2045,7 +1853,7 @@ ] }, { - "name": "va.gemini.azure", + "name": "va_gemini_azure", "auto_start": false, "nodes": [ { @@ -2109,26 +1917,6 @@ "extension_group": "transcriber", "addon": "message_collector", "name": "message_collector" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "default" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "llm" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "tts" - }, - { - "type": "extension_group", - "addon": "default_extension_group", - "name": "transcriber" } ], "connections": [ @@ -2244,7 +2032,7 @@ ] }, { - "name": "va.qwen.rag", + "name": "va_qwen_rag", "auto_start": false, "nodes": [ { @@ -2619,8 +2407,8 @@ ] }, { - "name": "va.openai.v2v", - "auto_start": false, + "name": "va_openai_v2v", + "auto_start": true, "nodes": [ { "type": "extension", @@ -2862,7 +2650,7 @@ ] }, { - "name": "va.openai.v2v.fish", + "name": "va_openai_v2v_fish", "auto_start": false, "nodes": [ { diff --git a/agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json b/agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json index d5dd00f9..7bcecbba 100644 --- a/agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json +++ b/agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/aliyun_text_embedding/manifest.json b/agents/ten_packages/extension/aliyun_text_embedding/manifest.json index ea02f9e0..bb46475e 100644 --- a/agents/ten_packages/extension/aliyun_text_embedding/manifest.json +++ b/agents/ten_packages/extension/aliyun_text_embedding/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/bedrock_llm_python/manifest.json b/agents/ten_packages/extension/bedrock_llm_python/manifest.json index 7a122b9e..aa924ce6 100644 --- a/agents/ten_packages/extension/bedrock_llm_python/manifest.json +++ b/agents/ten_packages/extension/bedrock_llm_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/bingsearch_tool_python/manifest.json b/agents/ten_packages/extension/bingsearch_tool_python/manifest.json index 3e5a4193..e25eff98 100644 --- a/agents/ten_packages/extension/bingsearch_tool_python/manifest.json +++ b/agents/ten_packages/extension/bingsearch_tool_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "package": { @@ -19,5 +19,55 @@ "README.md" ] }, - "api": {} + "api": { + "property": { + "api_key": { + "type": "string" + } + }, + "cmd_out": [ + { + "name": "tool_register", + "property": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "parameters": { + "type": "string" + } + }, + "required": [ + "name", + "description", + "parameters" + ], + "result": { + "property": { + "response": { + "type": "string" + } + } + } + } + ], + "cmd_in": [ + { + "name": "tool_call", + "property": { + "name": { + "type": "string" + }, + "args": { + "type": "string" + } + }, + "required": [ + "name" + ] + } + ] + } } \ No newline at end of file diff --git a/agents/ten_packages/extension/chat_transcriber_python/manifest.json b/agents/ten_packages/extension/chat_transcriber_python/manifest.json index f56247ba..34de9f78 100644 --- a/agents/ten_packages/extension/chat_transcriber_python/manifest.json +++ b/agents/ten_packages/extension/chat_transcriber_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/cosy_tts/manifest.json b/agents/ten_packages/extension/cosy_tts/manifest.json index 24ac6b09..56b42d79 100644 --- a/agents/ten_packages/extension/cosy_tts/manifest.json +++ b/agents/ten_packages/extension/cosy_tts/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/elevenlabs_tts/manifest.json b/agents/ten_packages/extension/elevenlabs_tts/manifest.json index b3fcc92f..1d57cb74 100644 --- a/agents/ten_packages/extension/elevenlabs_tts/manifest.json +++ b/agents/ten_packages/extension/elevenlabs_tts/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/elevenlabs_tts_python/manifest.json b/agents/ten_packages/extension/elevenlabs_tts_python/manifest.json index 7c0cc442..c712c10a 100644 --- a/agents/ten_packages/extension/elevenlabs_tts_python/manifest.json +++ b/agents/ten_packages/extension/elevenlabs_tts_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/fashionai/manifest.json b/agents/ten_packages/extension/fashionai/manifest.json index 521a7efb..b46866c4 100644 --- a/agents/ten_packages/extension/fashionai/manifest.json +++ b/agents/ten_packages/extension/fashionai/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "package": { diff --git a/agents/ten_packages/extension/file_chunker/manifest.json b/agents/ten_packages/extension/file_chunker/manifest.json index 1416c357..d74c75d2 100644 --- a/agents/ten_packages/extension/file_chunker/manifest.json +++ b/agents/ten_packages/extension/file_chunker/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/fish_audio_tts/manifest.json b/agents/ten_packages/extension/fish_audio_tts/manifest.json index d717fd53..d7422c13 100644 --- a/agents/ten_packages/extension/fish_audio_tts/manifest.json +++ b/agents/ten_packages/extension/fish_audio_tts/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/gemini_llm_python/manifest.json b/agents/ten_packages/extension/gemini_llm_python/manifest.json index 10f8f6ad..7f9702ea 100644 --- a/agents/ten_packages/extension/gemini_llm_python/manifest.json +++ b/agents/ten_packages/extension/gemini_llm_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/http_server_python/manifest.json b/agents/ten_packages/extension/http_server_python/manifest.json index e770edda..fb2e6e4a 100644 --- a/agents/ten_packages/extension/http_server_python/manifest.json +++ b/agents/ten_packages/extension/http_server_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "package": { diff --git a/agents/ten_packages/extension/interrupt_detector/manifest.json b/agents/ten_packages/extension/interrupt_detector/manifest.json index feb17b2c..4c602997 100644 --- a/agents/ten_packages/extension/interrupt_detector/manifest.json +++ b/agents/ten_packages/extension/interrupt_detector/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/interrupt_detector_python/manifest.json b/agents/ten_packages/extension/interrupt_detector_python/manifest.json index 97e540ad..bae13d83 100644 --- a/agents/ten_packages/extension/interrupt_detector_python/manifest.json +++ b/agents/ten_packages/extension/interrupt_detector_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/llama_index_chat_engine/manifest.json b/agents/ten_packages/extension/llama_index_chat_engine/manifest.json index 622d24fa..661fc070 100644 --- a/agents/ten_packages/extension/llama_index_chat_engine/manifest.json +++ b/agents/ten_packages/extension/llama_index_chat_engine/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/message_collector/manifest.json b/agents/ten_packages/extension/message_collector/manifest.json index 655a37a3..7085f2bf 100644 --- a/agents/ten_packages/extension/message_collector/manifest.json +++ b/agents/ten_packages/extension/message_collector/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "package": { diff --git a/agents/ten_packages/extension/openai_chatgpt/manifest.json b/agents/ten_packages/extension/openai_chatgpt/manifest.json index a0f97290..70ae2743 100644 --- a/agents/ten_packages/extension/openai_chatgpt/manifest.json +++ b/agents/ten_packages/extension/openai_chatgpt/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/openai_chatgpt_python/addon.py b/agents/ten_packages/extension/openai_chatgpt_python/addon.py index ee13b156..3c5400a9 100644 --- a/agents/ten_packages/extension/openai_chatgpt_python/addon.py +++ b/agents/ten_packages/extension/openai_chatgpt_python/addon.py @@ -18,5 +18,5 @@ class OpenAIChatGPTExtensionAddon(Addon): def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: - logger.info("OpenAIChatGPTExtensionAddon on_create_instance") + ten_env.log_info("OpenAIChatGPTExtensionAddon on_create_instance") ten_env.on_create_instance_done(OpenAIChatGPTExtension(name), context) diff --git a/agents/ten_packages/extension/openai_chatgpt_python/extension.py b/agents/ten_packages/extension/openai_chatgpt_python/extension.py index 4a5af8b8..15947832 100644 --- a/agents/ten_packages/extension/openai_chatgpt_python/extension.py +++ b/agents/ten_packages/extension/openai_chatgpt_python/extension.py @@ -7,8 +7,6 @@ # import asyncio import json -import random -import threading import traceback from .helper import AsyncEventEmitter, AsyncQueue, get_current_time, get_property_bool, get_property_float, get_property_int, get_property_string, parse_sentences, rgb2base64jpeg @@ -16,7 +14,7 @@ from ten import ( AudioFrame, VideoFrame, - Extension, + AsyncExtension, TenEnv, Cmd, StatusCode, @@ -54,7 +52,7 @@ TASK_TYPE_CHAT_COMPLETION_WITH_VISION = "chat_completion_with_vision" -class OpenAIChatGPTExtension(Extension): +class OpenAIChatGPTExtension(AsyncExtension): memory = [] max_memory_length = 10 openai_chatgpt = None @@ -81,20 +79,14 @@ class OpenAIChatGPTExtension(Extension): } ] - def on_init(self, ten_env: TenEnv) -> None: - logger.info("on_init") + async def on_init(self, ten_env: TenEnv) -> None: + ten_env.log_info("on_init") ten_env.on_init_done() - def on_start(self, ten_env: TenEnv) -> None: - logger.info("on_start") - - self.loop = asyncio.new_event_loop() - - def start_loop(): - asyncio.set_event_loop(self.loop) - self.loop.run_forever() - threading.Thread(target=start_loop, args=[]).start() + async def on_start(self, ten_env: TenEnv) -> None: + ten_env.log_info("on_start") + self.loop = asyncio.get_event_loop() self.loop.create_task(self._process_queue(ten_env)) # Prepare configuration @@ -106,7 +98,7 @@ def start_loop(): openai_chatgpt_config.api_key = get_property_string( ten_env, PROPERTY_API_KEY) if not openai_chatgpt_config.api_key: - logger.info(f"API key is missing, exiting on_start") + ten_env.log_info(f"API key is missing, exiting on_start") return # Optional properties @@ -139,39 +131,39 @@ def start_loop(): self.checking_vision_text_items = json.loads( checking_vision_text_items_str) except Exception as err: - logger.info( + ten_env.log_info( f"Error parsing {PROPERTY_CHECKING_VISION_TEXT_ITEMS}: {err}") self.users_count = 0 # Create instance try: self.openai_chatgpt = OpenAIChatGPT(openai_chatgpt_config) - logger.info( + ten_env.log_info( f"initialized with max_tokens: {openai_chatgpt_config.max_tokens}, model: {openai_chatgpt_config.model}") except Exception as err: - logger.info(f"Failed to initialize OpenAIChatGPT: {err}") + ten_env.log_info(f"Failed to initialize OpenAIChatGPT: {err}") ten_env.on_start_done() - def on_stop(self, ten_env: TenEnv) -> None: - logger.info("on_stop") + async def on_stop(self, ten_env: TenEnv) -> None: + ten_env.log_info("on_stop") # TODO: clean up resources ten_env.on_stop_done() - def on_deinit(self, ten_env: TenEnv) -> None: - logger.info("on_deinit") + async def on_deinit(self, ten_env: TenEnv) -> None: + ten_env.log_info("on_deinit") ten_env.on_deinit_done() - def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: + async def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: cmd_name = cmd.get_name() - logger.info(f"on_cmd name: {cmd_name}") + ten_env.log_info(f"on_cmd name: {cmd_name}") if cmd_name == CMD_IN_FLUSH: - asyncio.run_coroutine_threadsafe(self._flush_queue(), self.loop) + await self._flush_queue(ten_env) ten_env.send_cmd(Cmd.create(CMD_OUT_FLUSH), None) - logger.info("on_cmd sent flush") + ten_env.log_info("on_cmd sent flush") status_code, detail = StatusCode.OK, "success" elif cmd_name == CMD_IN_ON_USER_JOINED: self.users_count += 1 @@ -184,9 +176,9 @@ def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: output_data.set_property_bool( DATA_OUT_TEXT_DATA_PROPERTY_TEXT_END_OF_SEGMENT, True) ten_env.send_data(output_data) - logger.info(f"Greeting [{self.greeting}] sent") + ten_env.log_info(f"Greeting [{self.greeting}] sent") except Exception as err: - logger.info( + ten_env.log_info( f"Failed to send greeting [{self.greeting}]: {err}") status_code, detail = StatusCode.OK, "success" @@ -194,37 +186,36 @@ def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: self.users_count -= 1 status_code, detail = StatusCode.OK, "success" else: - logger.info(f"on_cmd unknown cmd: {cmd_name}") + ten_env.log_info(f"on_cmd unknown cmd: {cmd_name}") status_code, detail = StatusCode.ERROR, "unknown cmd" cmd_result = CmdResult.create(status_code) cmd_result.set_property_string("detail", detail) ten_env.return_result(cmd_result, cmd) - def on_data(self, ten_env: TenEnv, data: Data) -> None: + async def on_data(self, ten_env: TenEnv, data: Data) -> None: # Get the necessary properties is_final = get_property_bool(data, DATA_IN_TEXT_DATA_PROPERTY_IS_FINAL) input_text = get_property_string(data, DATA_IN_TEXT_DATA_PROPERTY_TEXT) if not is_final: - logger.info("ignore non-final input") + ten_env.log_info("ignore non-final input") return if not input_text: - logger.info("ignore empty text") + ten_env.log_info("ignore empty text") return - logger.info(f"OnData input text: [{input_text}]") + ten_env.log_info(f"OnData input text: [{input_text}]") # Start an asynchronous task for handling chat completion - asyncio.run_coroutine_threadsafe(self.queue.put( - [TASK_TYPE_CHAT_COMPLETION, input_text]), self.loop) + await self.queue.put([TASK_TYPE_CHAT_COMPLETION, input_text]) - def on_audio_frame(self, ten_env: TenEnv, audio_frame: AudioFrame) -> None: + async def on_audio_frame(self, ten_env: TenEnv, audio_frame: AudioFrame) -> None: # TODO: process pcm frame pass - def on_video_frame(self, ten_env: TenEnv, video_frame: VideoFrame) -> None: - # logger.info(f"OpenAIChatGPTExtension on_video_frame {frame.get_width()} {frame.get_height()}") + async def on_video_frame(self, ten_env: TenEnv, video_frame: VideoFrame) -> None: + # ten_env.log_info(f"OpenAIChatGPTExtension on_video_frame {frame.get_width()} {frame.get_height()}") self.image_data = video_frame.get_buf() self.image_width = video_frame.get_width() self.image_height = video_frame.get_height() @@ -241,23 +232,23 @@ async def _process_queue(self, ten_env: TenEnv): self._run_chatflow(ten_env, task_type, message, self.memory)) await self.current_task # Wait for the current task to finish or be cancelled except asyncio.CancelledError: - logger.info(f"Task cancelled: {message}") + ten_env.log_info(f"Task cancelled: {message}") - async def _flush_queue(self): + async def _flush_queue(self, ten_env: TenEnv): """Flushes the self.queue and cancels the current task.""" # Flush the queue using the new flush method await self.queue.flush() # Cancel the current task if one is running if self.current_task: - logger.info("Cancelling the current task during flush.") + ten_env.log_info("Cancelling the current task during flush.") self.current_task.cancel() async def _run_chatflow(self, ten_env: TenEnv, task_type: str, input_text: str, memory): """Run the chatflow asynchronously.""" memory_cache = [] try: - logger.info(f"for input text: [{input_text}] memory: {memory}") + ten_env.log_info(f"for input text: [{input_text}] memory: {memory}") message = None tools = None @@ -282,7 +273,7 @@ async def _run_chatflow(self, ten_env: TenEnv, task_type: str, input_text: str, {"type": "image_url", "image_url": {"url": url}}, ], } - logger.info(f"msg with vision data: {message}") + ten_env.log_info(f"msg with vision data: {message}") self.sentence_fragment = "" @@ -291,7 +282,7 @@ async def _run_chatflow(self, ten_env: TenEnv, task_type: str, input_text: str, # Create an async listener to handle tool calls and content updates async def handle_tool_call(tool_call): - logger.info(f"tool_call: {tool_call}") + ten_env.log_info(f"tool_call: {tool_call}") if tool_call.function.name == "get_vision_image": # Append the vision image to the last assistant message await self.queue.put([TASK_TYPE_CHAT_COMPLETION_WITH_VISION, input_text], True) @@ -321,7 +312,7 @@ async def handle_content_finished(full_content: str): # Wait for the content to be finished await content_finished_event.wait() except asyncio.CancelledError: - logger.info(f"Task cancelled: {input_text}") + ten_env.log_info(f"Task cancelled: {input_text}") except Exception as e: logger.error( f"Error in chat_completion: {traceback.format_exc()} for input text: {input_text}") @@ -345,10 +336,10 @@ def _send_data(self, ten_env: TenEnv, sentence: str, end_of_segment: bool): DATA_OUT_TEXT_DATA_PROPERTY_TEXT_END_OF_SEGMENT, end_of_segment ) ten_env.send_data(output_data) - logger.info( + ten_env.log_info( f"{'end of segment ' if end_of_segment else ''}sent sentence [{sentence}]" ) except Exception as err: - logger.info( + ten_env.log_info( f"send sentence [{sentence}] failed, err: {err}" ) diff --git a/agents/ten_packages/extension/openai_chatgpt_python/manifest.json b/agents/ten_packages/extension/openai_chatgpt_python/manifest.json index c71638ac..68423776 100644 --- a/agents/ten_packages/extension/openai_chatgpt_python/manifest.json +++ b/agents/ten_packages/extension/openai_chatgpt_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "package": { diff --git a/agents/ten_packages/extension/openai_v2v_python/manifest.json b/agents/ten_packages/extension/openai_v2v_python/manifest.json index 55b91ece..feda88b5 100644 --- a/agents/ten_packages/extension/openai_v2v_python/manifest.json +++ b/agents/ten_packages/extension/openai_v2v_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "package": { diff --git a/agents/ten_packages/extension/polly_tts/manifest.json b/agents/ten_packages/extension/polly_tts/manifest.json index 932c3b47..6c90bb94 100644 --- a/agents/ten_packages/extension/polly_tts/manifest.json +++ b/agents/ten_packages/extension/polly_tts/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/qwen_llm_python/manifest.json b/agents/ten_packages/extension/qwen_llm_python/manifest.json index bdad7d87..76dcb560 100644 --- a/agents/ten_packages/extension/qwen_llm_python/manifest.json +++ b/agents/ten_packages/extension/qwen_llm_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/transcribe_asr_python/manifest.json b/agents/ten_packages/extension/transcribe_asr_python/manifest.json index 5950e3f7..fbdf7aa0 100644 --- a/agents/ten_packages/extension/transcribe_asr_python/manifest.json +++ b/agents/ten_packages/extension/transcribe_asr_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "api": { diff --git a/agents/ten_packages/extension/weatherapi_tool_python/manifest.json b/agents/ten_packages/extension/weatherapi_tool_python/manifest.json index ea553824..7e000c93 100644 --- a/agents/ten_packages/extension/weatherapi_tool_python/manifest.json +++ b/agents/ten_packages/extension/weatherapi_tool_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.2" + "version": "0.3" } ], "package": { diff --git a/demo/src/app/api/agents/start/graph.tsx b/demo/src/app/api/agents/start/graph.tsx index a1fda36c..6d0de2c6 100644 --- a/demo/src/app/api/agents/start/graph.tsx +++ b/demo/src/app/api/agents/start/graph.tsx @@ -90,7 +90,7 @@ export const getGraphProperties = ( } } - if (graphName == "camera.va.openai.azure") { + if (graphName == "camera_va_openai_azure") { return { "agora_rtc": { "agora_asr_language": language, @@ -105,7 +105,7 @@ export const getGraphProperties = ( "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] } } - } else if (graphName == "va.openai.v2v") { + } else if (graphName == "va_openai_v2v") { return { "openai_v2v_python": { "model": "gpt-4o-realtime-preview", @@ -116,7 +116,7 @@ export const getGraphProperties = ( "greeting": greeting, } } - } else if (graphName == "va.openai.v2v.fish") { + } else if (graphName == "va_openai_v2v_fish") { return { "openai_v2v_python": { "model": "gpt-4o-realtime-preview", @@ -130,7 +130,7 @@ export const getGraphProperties = ( "agora_asr_language": language, }, } - } else if (graphName == "va.openai.azure") { + } else if (graphName == "va_openai_azure") { return { "agora_rtc": { "agora_asr_language": language, @@ -145,7 +145,7 @@ export const getGraphProperties = ( "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] } } - } else if (graphName == "va.qwen.rag") { + } else if (graphName == "va_qwen_rag") { return { "agora_rtc": { "agora_asr_language": language, diff --git a/demo/src/common/constant.ts b/demo/src/common/constant.ts index 6240fbd5..9e62df20 100644 --- a/demo/src/common/constant.ts +++ b/demo/src/common/constant.ts @@ -35,28 +35,28 @@ export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [ export const GRAPH_OPTIONS: GraphOptionItem[] = [ { label: "Voice Agent - OpenAI LLM + Azure TTS", - value: "va.openai.azure" + value: "va_openai_azure" }, { label: "Voice Agent with Vision - OpenAI LLM + Azure TTS", - value: "camera.va.openai.azure" + value: "camera_va_openai_azure" }, // { // label: "Voice Agent with Knowledge - RAG + Qwen LLM + Cosy TTS", - // value: "va.qwen.rag" + // value: "va_qwen_rag" // }, { label: "Voice Agent with OpenAI Realtime API (Beta)", - value: "va.openai.v2v" + value: "va_openai_v2v" }, { label: "Voice Agent with OpenAI Realtime API (Beta) + FishAudio TTS", - value: "va.openai.v2v.fish" + value: "va_openai_v2v_fish" } ] export const isRagGraph = (graphName: string) => { - return graphName === "va.qwen.rag" + return graphName === "va_qwen_rag" } export const VOICE_OPTIONS: VoiceOptionItem[] = [ diff --git a/demo/src/store/reducers/global.ts b/demo/src/store/reducers/global.ts index 25a5e29b..bd093fa8 100644 --- a/demo/src/store/reducers/global.ts +++ b/demo/src/store/reducers/global.ts @@ -23,7 +23,7 @@ const getInitialState = (): InitialState => { language: "en-US", voiceType: "male", chatItems: [], - graphName: "va.openai.v2v", + graphName: "va_openai_v2v", agentSettings: DEFAULT_AGENT_SETTINGS, } } diff --git a/docker-compose.yml b/docker-compose.yml index 46fcc8d0..5a1db726 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: ten_agent_dev: - image: ghcr.io/ten-framework/ten_agent_build:0.1.0 + image: ghcr.io/ten-framework/ten_agent_build:0.2.1 container_name: ten_agent_dev platform: linux/amd64 tty: true @@ -19,7 +19,7 @@ services: networks: - ten_agent_network ten_agent_playground: - image: ghcr.io/ten-framework/ten_agent_playground + image: ghcr.io/ten-framework/ten_agent_playground:0.5.0-56-g0536dbb container_name: ten_agent_playground restart: always ports: @@ -30,7 +30,7 @@ services: - AGENT_SERVER_URL=http://ten_agent_dev:8080 - TEN_DEV_SERVER_URL=http://ten_agent_dev:49483 ten_agent_demo: - image: ghcr.io/ten-framework/ten_agent_demo:0.5.0-50-gf9d5d4e + image: ghcr.io/ten-framework/ten_agent_demo:0.5.0-56-g0536dbb container_name: ten_agent_demo restart: always ports: diff --git a/playground/src/apis/routes.tsx b/playground/src/apis/routes.tsx index 64f16fb6..2f1affc3 100644 --- a/playground/src/apis/routes.tsx +++ b/playground/src/apis/routes.tsx @@ -6,7 +6,7 @@ const { AGENT_SERVER_URL } = process.env; // Check if environment variables are available if (!AGENT_SERVER_URL) { - throw "Environment variables AGENT_SERVER_URL are not available"; + throw "Environment variables AGENT_SERVER_URL are not available"; } @@ -78,7 +78,7 @@ export const getGraphProperties = (graphName: string, language: string, voiceTyp } } - if (graphName == "camera.va.openai.azure") { + if (graphName == "camera_va_openai_azure") { return { "agora_rtc": { "agora_asr_language": language, @@ -91,7 +91,7 @@ export const getGraphProperties = (graphName: string, language: string, voiceTyp "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] } } - } else if (graphName == "va.openai.azure") { + } else if (graphName == "va_openai_azure") { return { "agora_rtc": { "agora_asr_language": language, @@ -104,7 +104,7 @@ export const getGraphProperties = (graphName: string, language: string, voiceTyp "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] } } - } else if (graphName == "va.qwen.rag") { + } else if (graphName == "va_qwen_rag") { return { "agora_rtc": { "agora_asr_language": language, @@ -118,42 +118,42 @@ export const getGraphProperties = (graphName: string, language: string, voiceTyp } export async function startAgent(request: NextRequest) { - try{ - const body = await request.json(); - const { - request_id, - channel_name, - user_uid, - graph_name, - language, - voice_type, - } = body; + try { + const body = await request.json(); + const { + request_id, + channel_name, + user_uid, + graph_name, + language, + voice_type, + } = body; - // Send a POST request to start the agent - const response = await fetch(`${AGENT_SERVER_URL}/start`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - request_id, - channel_name, - user_uid, - graph_name, - // Get the graph properties based on the graph name, language, and voice type - properties: getGraphProperties(graph_name, language, voice_type), - }), - }); + // Send a POST request to start the agent + const response = await fetch(`${AGENT_SERVER_URL}/start`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + request_id, + channel_name, + user_uid, + graph_name, + // Get the graph properties based on the graph name, language, and voice type + properties: getGraphProperties(graph_name, language, voice_type), + }), + }); - const responseData = await response.json(); + const responseData = await response.json(); - return NextResponse.json(responseData, { status: response.status }); - } catch (error) { - if (error instanceof Response) { - const errorData = await error.json(); - return NextResponse.json(errorData, { status: error.status }); - } else { - return NextResponse.json({ code: "1", data: null, msg: "Internal Server Error" }, { status: 500 }); + return NextResponse.json(responseData, { status: response.status }); + } catch (error) { + if (error instanceof Response) { + const errorData = await error.json(); + return NextResponse.json(errorData, { status: error.status }); + } else { + return NextResponse.json({ code: "1", data: null, msg: "Internal Server Error" }, { status: 500 }); + } } - } } diff --git a/playground/src/common/constant.ts b/playground/src/common/constant.ts index 3874bad7..14b837d2 100644 --- a/playground/src/common/constant.ts +++ b/playground/src/common/constant.ts @@ -28,20 +28,20 @@ export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [ export const GRAPH_OPTIONS: GraphOptionItem[] = [ { label: "Voice Agent - OpenAI LLM + Azure TTS", - value: "va.openai.azure" + value: "va_openai_azure" }, { label: "Voice Agent with Vision - OpenAI LLM + Azure TTS", - value: "camera.va.openai.azure" + value: "camera_va_openai_azure" }, //{ // label: "Voice Agent with Knowledge - RAG + Qwen LLM + Cosy TTS", - // value: "va.qwen.rag" + // value: "va_qwen_rag" // }, ] export const isRagGraph = (graphName: string) => { - return graphName === "va.qwen.rag" + return graphName === "va_qwen_rag" } export const VOICE_OPTIONS: VoiceOptionItem[] = [ diff --git a/playground/src/store/reducers/global.ts b/playground/src/store/reducers/global.ts index dbdbff3b..eff3627e 100644 --- a/playground/src/store/reducers/global.ts +++ b/playground/src/store/reducers/global.ts @@ -25,7 +25,7 @@ const getInitialState = (): InitialState => { language: "en-US", voiceType: "male", chatItems: [], - graphName: "camera.va.openai.azure", + graphName: "camera_va_openai_azure", graphs: [], extensions: {}, extensionMetadata: {}, diff --git a/server/README.md b/server/README.md index 9ecf4132..dc589fcf 100644 --- a/server/README.md +++ b/server/README.md @@ -29,7 +29,7 @@ curl 'http://localhost:8080/start' \ "request_id": "c1912182-924c-4d15-a8bb-85063343077c", "channel_name": "test", "user_uid": 176573, - "graph_name": "camera.va.openai.azure", + "graph_name": "camera_va_openai_azure", "properties": { "openai_chatgpt": { "model": "gpt-4o"