From 2f32a6915b91c69d2a763daaf13ae3d52d2149a4 Mon Sep 17 00:00:00 2001 From: Zhang Qianze Date: Tue, 26 Nov 2024 05:05:34 +0800 Subject: [PATCH 01/28] feat: change default graph to contain deepgram & fish only --- README.md | 2 +- Taskfile.yml | 2 +- agents/examples/default/manifest.json | 42 +++ agents/examples/default/property.json | 251 ++++++++++++ agents/examples/experimental/property.json | 399 -------------------- playground/src/components/Chat/ChatCard.tsx | 2 +- playground/src/components/Layout/Action.tsx | 1 + playground/src/store/reducers/global.ts | 2 +- 8 files changed, 298 insertions(+), 403 deletions(-) create mode 100644 agents/examples/default/manifest.json create mode 100644 agents/examples/default/property.json diff --git a/README.md b/README.md index f369b2b3..08f439e3 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ Open up a separate terminal window, enter the container and build the agent: ```bash docker exec -it ten_agent_dev bash -task use AGENT=agents/examples/demo +task use AGENT=agents/examples/default ``` #### 5. Start the server diff --git a/Taskfile.yml b/Taskfile.yml index e27827c2..aeccadcf 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -16,7 +16,7 @@ tasks: use: desc: use agent, default 'agents/examples/demo' vars: - AGENT: '{{.AGENT| default "agents/examples/demo"}}' + AGENT: '{{.AGENT| default "agents/examples/default"}}' cmds: - ln -sf {{.USER_WORKING_DIR}}/{{.AGENT}}/manifest.json ./agents/ - ln -sf {{.USER_WORKING_DIR}}/{{.AGENT}}/property.json ./agents/ diff --git a/agents/examples/default/manifest.json b/agents/examples/default/manifest.json new file mode 100644 index 00000000..ab4316ed --- /dev/null +++ b/agents/examples/default/manifest.json @@ -0,0 +1,42 @@ +{ + "type": "app", + "name": "agent_demo", + "version": "0.4.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_go", + "version": "0.4" + }, + { + "type": "extension", + "name": "py_init_extension_cpp", + "version": "0.4" + }, + { + "type": "extension", + "name": "agora_rtc", + "version": "=0.9.0-rc1" + }, + { + "type": "extension", + "name": "agora_sess_ctrl", + "version": "0.3.0-rc1" + }, + { + "type": "system", + "name": "azure_speech_sdk", + "version": "1.38.0" + }, + { + "type": "extension", + "name": "azure_tts", + "version": "=0.6.0" + }, + { + "type": "extension", + "name": "agora_rtm", + "version": "=0.3.0" + } + ] +} \ No newline at end of file diff --git a/agents/examples/default/property.json b/agents/examples/default/property.json new file mode 100644 index 00000000..7126da40 --- /dev/null +++ b/agents/examples/default/property.json @@ -0,0 +1,251 @@ +{ + "_ten": { + "log_level": 3, + "predefined_graphs": [ + { + "name": "va_deepgram_openai_fish", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": false, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "asr", + "addon": "deepgram_asr_python", + "name": "deepgram_asr", + "property": { + "api_key": "${env:DEEPGRAM_API_KEY}", + "language": "en-US", + "model": "nova-2", + "sample_rate": 16000 + } + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt_python", + "name": "openai_chatgpt", + "property": { + "base_url": "", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "gpt-4o-mini", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL|}", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "fish_audio_tts", + "name": "fish_audio_tts", + "property": { + "api_key": "${env:FISH_AUDIO_TTS_KEY}", + "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", + "optimize_streaming_latency": true, + "request_timeout_seconds": 30, + "base_url": "https://api.fish.audio" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + }, + { + "name": "on_connection_failure", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + } + ] + }, + { + "extension_group": "asr", + "extension": "deepgram_asr", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "fish_audio_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "fish_audio_tts" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "fish_audio_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/agents/examples/experimental/property.json b/agents/examples/experimental/property.json index b532fea5..9e21bc7f 100644 --- a/agents/examples/experimental/property.json +++ b/agents/examples/experimental/property.json @@ -198,125 +198,6 @@ } ] }, - { - "name": "va_simple_azure_openai_v2v", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000 - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:AZURE_OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "voice": "alloy", - "language": "en-US", - "server_vad": true, - "dump": true, - "history": 10, - "vendor": "azure", - "base_uri": "${env:AZURE_OPENAI_REALTIME_BASE_URI}", - "path": "/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview", - "system_message": "" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "openai_v2v_python", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, { "name": "va_openai_11labs", "auto_start": false, @@ -2409,286 +2290,6 @@ } ] }, - { - "name": "va_simple_openai_v2v", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000 - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "voice": "alloy", - "language": "en-US", - "server_vad": true, - "dump": true, - "history": 10 - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "openai_v2v_python", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "va_simple_openai_v2v_fish", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "audio_out": false, - "input_transcript": false, - "language": "en-US", - "server_vad": true, - "dump": true, - "history": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "fish_audio_tts", - "name": "fish_audio_tts", - "property": { - "api_key": "${env:FISH_AUDIO_TTS_KEY}", - "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", - "optimize_streaming_latency": true, - "request_timeout_seconds": 30, - "base_url": "https://api.fish.audio" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "openai_v2v_python", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - }, - { - "extension_group": "tts", - "extension": "fish_audio_tts" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "fish_audio_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "fish_audio_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, { "name": "va_openai_v2v_storage", "auto_start": false, diff --git a/playground/src/components/Chat/ChatCard.tsx b/playground/src/components/Chat/ChatCard.tsx index 517b4b17..b4828c65 100644 --- a/playground/src/components/Chat/ChatCard.tsx +++ b/playground/src/components/Chat/ChatCard.tsx @@ -100,7 +100,7 @@ export default function ChatCard(props: { className?: string }) { }, []); React.useEffect(() => { - if (!extensions[graphName]) { + if (graphName !== "" && !extensions[graphName]) { apiGetNodes(graphName).then((res: any) => { let nodes = res["data"]; let nodesMap: Record = {}; diff --git a/playground/src/components/Layout/Action.tsx b/playground/src/components/Layout/Action.tsx index d96fddbf..913ee170 100644 --- a/playground/src/components/Layout/Action.tsx +++ b/playground/src/components/Layout/Action.tsx @@ -148,6 +148,7 @@ export default function Action(props: { className?: string }) { onClick={onClickConnect} variant={!agentConnected ? "default" : "destructive"} size="sm" + disabled={graphName === ""} className="w-fit min-w-24" loading={loading} svgProps={{ className: "h-4 w-4 text-muted-foreground" }} diff --git a/playground/src/store/reducers/global.ts b/playground/src/store/reducers/global.ts index 9f6e6ee0..112128a3 100644 --- a/playground/src/store/reducers/global.ts +++ b/playground/src/store/reducers/global.ts @@ -43,7 +43,7 @@ const getInitialState = (): InitialState => { language: "en-US", voiceType: "male", chatItems: [], - graphName: "camera_va_openai_azure", + graphName: "", graphs: [], extensions: {}, overridenProperties: {}, From a4a91af556514a9133656212d52ae7524fa397e1 Mon Sep 17 00:00:00 2001 From: Ethan Zhang Date: Tue, 26 Nov 2024 12:56:08 +0800 Subject: [PATCH 02/28] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2f681093..dc8cf759 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: networks: - ten_agent_network ten_agent_playground: - image: ghcr.io/ten-framework/ten_agent_playground:0.6.1-15-g1160ef4 + image: ghcr.io/ten-framework/ten_agent_playground:0.6.1-17-g2f32a69 container_name: ten_agent_playground restart: always ports: From 8e9431b28d90840d294f82f446cb0023bef81525 Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Tue, 26 Nov 2024 23:27:38 +0800 Subject: [PATCH 03/28] feat: initial change --- agents/examples/default/property.json | 42 +++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/agents/examples/default/property.json b/agents/examples/default/property.json index 7126da40..0999b5bb 100644 --- a/agents/examples/default/property.json +++ b/agents/examples/default/property.json @@ -3,7 +3,7 @@ "log_level": 3, "predefined_graphs": [ { - "name": "va_deepgram_openai_fish", + "name": "voice_assistant", "auto_start": false, "nodes": [ { @@ -30,9 +30,9 @@ }, { "type": "extension", - "extension_group": "asr", + "extension_group": "stt", "addon": "deepgram_asr_python", - "name": "deepgram_asr", + "name": "stt", "property": { "api_key": "${env:DEEPGRAM_API_KEY}", "language": "en-US", @@ -44,7 +44,7 @@ "type": "extension", "extension_group": "chatgpt", "addon": "openai_chatgpt_python", - "name": "openai_chatgpt", + "name": "llm", "property": { "base_url": "", "api_key": "${env:OPENAI_API_KEY}", @@ -61,7 +61,7 @@ "type": "extension", "extension_group": "tts", "addon": "fish_audio_tts", - "name": "fish_audio_tts", + "name": "tts", "property": { "api_key": "${env:FISH_AUDIO_TTS_KEY}", "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", @@ -92,8 +92,8 @@ "name": "pcm_frame", "dest": [ { - "extension_group": "asr", - "extension": "deepgram_asr" + "extension_group": "stt", + "extension": "stt" } ] } @@ -103,8 +103,8 @@ "name": "on_user_joined", "dest": [ { - "extension_group": "asr", - "extension": "deepgram_asr" + "extension_group": "stt", + "extension": "stt" } ] }, @@ -112,8 +112,8 @@ "name": "on_user_left", "dest": [ { - "extension_group": "asr", - "extension": "deepgram_asr" + "extension_group": "stt", + "extension": "stt" } ] }, @@ -121,16 +121,16 @@ "name": "on_connection_failure", "dest": [ { - "extension_group": "asr", - "extension": "deepgram_asr" + "extension_group": "stt", + "extension": "stt" } ] } ] }, { - "extension_group": "asr", - "extension": "deepgram_asr", + "extension_group": "stt", + "extension": "stt", "data": [ { "name": "text_data", @@ -149,14 +149,14 @@ }, { "extension_group": "chatgpt", - "extension": "openai_chatgpt", + "extension": "llm", "data": [ { "name": "text_data", "dest": [ { "extension_group": "tts", - "extension": "fish_audio_tts" + "extension": "tts" }, { "extension_group": "transcriber", @@ -171,7 +171,7 @@ "dest": [ { "extension_group": "tts", - "extension": "fish_audio_tts" + "extension": "tts" } ] } @@ -194,7 +194,7 @@ }, { "extension_group": "tts", - "extension": "fish_audio_tts", + "extension": "tts", "audio_frame": [ { "name": "pcm_frame", @@ -227,7 +227,7 @@ "dest": [ { "extension_group": "chatgpt", - "extension": "openai_chatgpt" + "extension": "llm" } ] } @@ -238,7 +238,7 @@ "dest": [ { "extension_group": "chatgpt", - "extension": "openai_chatgpt" + "extension": "llm" } ] } From f476789b54edad1f1c7ccf152c039e05664b5320 Mon Sep 17 00:00:00 2001 From: Zhang Qianze Date: Wed, 27 Nov 2024 12:31:44 +0800 Subject: [PATCH 04/28] feat: add module setting dialog --- playground/src/common/hooks.ts | 6 + playground/src/components/Chat/ChatCard.tsx | 6 +- .../src/components/Chat/ChatCfgSelect.tsx | 224 +++++++++++++++++- playground/src/components/Layout/Action.tsx | 1 - 4 files changed, 232 insertions(+), 5 deletions(-) diff --git a/playground/src/common/hooks.ts b/playground/src/common/hooks.ts index 87bb7c1f..41e031f3 100644 --- a/playground/src/common/hooks.ts +++ b/playground/src/common/hooks.ts @@ -163,3 +163,9 @@ export const useGraphExtensions = () => { return graphExtensions; }; + + +export const useExtensionsMetadataNames = (): string[] => { + const metadata = useAppSelector((state) => state.global.extensionMetadata); + return Object.keys(metadata) +}; \ No newline at end of file diff --git a/playground/src/components/Chat/ChatCard.tsx b/playground/src/components/Chat/ChatCard.tsx index b4828c65..16392342 100644 --- a/playground/src/components/Chat/ChatCard.tsx +++ b/playground/src/components/Chat/ChatCard.tsx @@ -4,7 +4,8 @@ import * as React from "react"; import { cn } from "@/lib/utils"; import { RemoteGraphSelect, - RemoteGraphCfgSheet, + RemoteModuleCfgSheet, + RemotePropertyCfgSheet, } from "@/components/Chat/ChatCfgSelect"; import PdfSelect from "@/components/Chat/PdfSelect"; import { @@ -166,7 +167,8 @@ export default function ChatCard(props: { className?: string }) { {/* Action Bar */}
- + + {isRagGraph(graphName) && }
{/* Chat messages would go here */} diff --git a/playground/src/components/Chat/ChatCfgSelect.tsx b/playground/src/components/Chat/ChatCfgSelect.tsx index 499ec1b3..ef2ab546 100644 --- a/playground/src/components/Chat/ChatCfgSelect.tsx +++ b/playground/src/components/Chat/ChatCfgSelect.tsx @@ -42,6 +42,7 @@ import { useAppSelector, GRAPH_OPTIONS, useGraphExtensions, + useExtensionsMetadataNames, } from "@/common" import type { Language } from "@/types" import { @@ -50,7 +51,7 @@ import { setOverridenPropertiesByGraph, } from "@/store/reducers/global" import { cn } from "@/lib/utils" -import { SettingsIcon, LoaderCircleIcon } from "lucide-react" +import { SettingsIcon, LoaderCircleIcon, BoxesIcon } from "lucide-react" import { zodResolver } from "@hookform/resolvers/zod" import { useForm } from "react-hook-form" import { z } from "zod" @@ -93,7 +94,89 @@ export function RemoteGraphSelect() { ) } -export function RemoteGraphCfgSheet() { +export function RemoteModuleCfgSheet() { + const dispatch = useAppDispatch() + const graphExtensions = useGraphExtensions() + const graphName = useAppSelector((state) => state.global.graphName) + const extensionMetadata = useAppSelector( + (state) => state.global.extensionMetadata, + ) + const extensionMetadataNames = useExtensionsMetadataNames() + const overridenProperties = useAppSelector( + (state) => state.global.overridenProperties, + ) + + const sttExtensionNames = React.useMemo(() => { + return extensionMetadataNames.filter((item) => + item.includes("stt") || item.includes("asr"), + ) + }, [extensionMetadata]) + + const llmExtensionNames = React.useMemo(() => { + return extensionMetadataNames.filter((item) => + item.includes("llm"), + ) + }, [extensionMetadata]) + + const ttsExtensionNames = React.useMemo(() => { + return extensionMetadataNames.filter((item) => + item.includes("tts"), + ) + }, [extensionMetadata]) + + console.log(extensionMetadataNames) + + const metadata = { + STT: { type: "string", options: sttExtensionNames }, + LLM: { type: "string", options: llmExtensionNames }, + TTS: { type: "string", options: ttsExtensionNames }, + } + + const initialData = { + STT: null, + LLM: null, + TTS: null, + } + + return ( + + + + + + + Module Picker + + You can adjust extension modules here, the values will be + overridden when the agent starts using "Connect." Note that this + won't modify the property.json file. + + + +
+ {}} + /> +
+ + {/* + + + + */} + + + ) +} + +export function RemotePropertyCfgSheet() { const dispatch = useAppDispatch() const graphExtensions = useGraphExtensions() const graphName = useAppSelector((state) => state.global.graphName) @@ -206,6 +289,143 @@ const convertToType = (value: any, type: string) => { } } +const GraphModuleCfgForm = ({ + initialData, + metadata, + onUpdate, +}: { + initialData: Record + metadata: Record + onUpdate: (data: Record) => void +}) => { + const formSchema = z.object({ + STT: z.string().nullable(), + LLM: z.string().nullable(), + TTS: z.string().nullable(), + }) + + const form = useForm>({ + resolver: zodResolver(formSchema), + defaultValues: initialData, + }) + + const onSubmit = (data: z.infer) => { + onUpdate(data) + } + + return ( +
+ + {/* STT Section */} +
+

STT (Speech to Text)

+ ( + + Speech-to-Text + + + + + )} + /> +
+ + {/* LLM Section */} +
+

LLM (Large Language Model)

+ ( + + Large Language Model + + + + + )} + /> +
+ + {/* TTS Section */} +
+

TTS (Text to Speech)

+ ( + + Text-to-Speech + + + + + )} + /> +
+ + {/* Submit Button */} + +
+ + ) +} + + const GraphCfgForm = ({ initialData, metadata, diff --git a/playground/src/components/Layout/Action.tsx b/playground/src/components/Layout/Action.tsx index 913ee170..9405f232 100644 --- a/playground/src/components/Layout/Action.tsx +++ b/playground/src/components/Layout/Action.tsx @@ -139,7 +139,6 @@ export default function Action(props: { className?: string }) { ))} - yarn {/* -- Action Button */} From dfed145f937d0610768908ffb42f10a8eab08dea Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Thu, 28 Nov 2024 14:39:59 +0800 Subject: [PATCH 05/28] add addon property --- .../src/components/Chat/ChatCfgSelect.tsx | 18 +++++++++--------- server/internal/http_server.go | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/playground/src/components/Chat/ChatCfgSelect.tsx b/playground/src/components/Chat/ChatCfgSelect.tsx index ef2ab546..ef8c38a6 100644 --- a/playground/src/components/Chat/ChatCfgSelect.tsx +++ b/playground/src/components/Chat/ChatCfgSelect.tsx @@ -299,9 +299,9 @@ const GraphModuleCfgForm = ({ onUpdate: (data: Record) => void }) => { const formSchema = z.object({ - STT: z.string().nullable(), - LLM: z.string().nullable(), - TTS: z.string().nullable(), + stt: z.string().nullable(), + llm: z.string().nullable(), + tts: z.string().nullable(), }) const form = useForm>({ @@ -321,7 +321,7 @@ const GraphModuleCfgForm = ({

STT (Speech to Text)

( Speech-to-Text @@ -334,7 +334,7 @@ const GraphModuleCfgForm = ({ - {metadata["STT"].options.map((option) => ( + {metadata["stt"].options.map((option) => ( {option} @@ -352,7 +352,7 @@ const GraphModuleCfgForm = ({

LLM (Large Language Model)

( Large Language Model @@ -365,7 +365,7 @@ const GraphModuleCfgForm = ({ - {metadata["LLM"].options.map((option) => ( + {metadata["llm"].options.map((option) => ( {option} @@ -383,7 +383,7 @@ const GraphModuleCfgForm = ({

TTS (Text to Speech)

( Text-to-Speech @@ -396,7 +396,7 @@ const GraphModuleCfgForm = ({ - {metadata["TTS"].options.map((option) => ( + {metadata["tts"].options.map((option) => ( {option} diff --git a/server/internal/http_server.go b/server/internal/http_server.go index c9935f72..cc9a29fc 100644 --- a/server/internal/http_server.go +++ b/server/internal/http_server.go @@ -54,6 +54,7 @@ type StartReq struct { Token string `json:"token,omitempty"` WorkerHttpServerPort int32 `json:"worker_http_server_port,omitempty"` Properties map[string]map[string]interface{} `json:"properties,omitempty"` + Addons map[string]interface{} `json:"addons,omitempty"` QuitTimeoutSeconds int `json:"timeout,omitempty"` } @@ -449,6 +450,20 @@ func (s *HttpServer) processProperty(req *StartReq) (propertyJsonFile string, lo graphMap["auto_start"] = true } + // Process Addons property + for addonKey, addonValue := range req.Addons { + for _, graph := range newGraphs { + graphMap, _ := graph.(map[string]interface{}) + nodes, _ := graphMap["nodes"].([]interface{}) + for _, node := range nodes { + nodeMap, _ := node.(map[string]interface{}) + if nodeMap["name"] == addonKey { + nodeMap["addon"] = addonValue + } + } + } + } + // Set additional properties to property.json for extensionName, props := range req.Properties { if extensionName != "" { From a4cf49f348687e77f6949f6fb9ba83e90c5171fa Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Fri, 29 Nov 2024 03:42:18 +0800 Subject: [PATCH 06/28] feat: initial module picker --- Taskfile.yml | 1 - .../elevenlabs_tts_python/extension.py | 3 +- .../elevenlabs_tts_python/property.json | 11 +- .../extension/fish_audio_tts/property.json | 8 +- playground/src/apis/routes.tsx | 159 ------ playground/src/common/graph.ts | 490 ++++++++++++++++++ playground/src/common/hooks.ts | 9 +- playground/src/common/request.ts | 33 -- playground/src/components/Chat/ChatCard.tsx | 47 +- .../src/components/Chat/ChatCfgSelect.tsx | 116 +++-- playground/src/components/Layout/Action.tsx | 2 +- .../src/components/authInitializer/index.tsx | 3 + playground/src/middleware.tsx | 40 +- playground/src/store/reducers/global.ts | 36 +- server/internal/code.go | 2 + server/internal/http_server.go | 58 ++- 16 files changed, 687 insertions(+), 331 deletions(-) delete mode 100644 playground/src/apis/routes.tsx create mode 100644 playground/src/common/graph.ts diff --git a/Taskfile.yml b/Taskfile.yml index aeccadcf..e401461d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -49,7 +49,6 @@ tasks: build-server: desc: build server dir: ./server - internal: true cmds: - go mod tidy && go mod download && go build -o bin/api main.go diff --git a/agents/ten_packages/extension/elevenlabs_tts_python/extension.py b/agents/ten_packages/extension/elevenlabs_tts_python/extension.py index 01dd44b3..8e9f924b 100644 --- a/agents/ten_packages/extension/elevenlabs_tts_python/extension.py +++ b/agents/ten_packages/extension/elevenlabs_tts_python/extension.py @@ -45,9 +45,10 @@ async def on_deinit(self, ten_env: AsyncTenEnv) -> None: async def on_request_tts(self, ten_env: AsyncTenEnv, input_text: str, end_of_segment: bool) -> None: audio_stream = await self.client.text_to_speech_stream(input_text) - + ten_env.log_info(f"on_request_tts: {input_text}") async for audio_data in audio_stream: self.send_audio_out(ten_env, audio_data) + ten_env.log_info(f"on_request_tts: {input_text} done") async def on_cancel_tts(self, ten_env: AsyncTenEnv) -> None: return await super().on_cancel_tts(ten_env) \ No newline at end of file diff --git a/agents/ten_packages/extension/elevenlabs_tts_python/property.json b/agents/ten_packages/extension/elevenlabs_tts_python/property.json index 9e26dfee..a17ebff8 100644 --- a/agents/ten_packages/extension/elevenlabs_tts_python/property.json +++ b/agents/ten_packages/extension/elevenlabs_tts_python/property.json @@ -1 +1,10 @@ -{} \ No newline at end of file +{ + "api_key": "${env:ELEVENLABS_TTS_KEY}", + "model_id": "eleven_multilingual_v2", + "optimize_streaming_latency": 0, + "request_timeout_seconds": 30, + "similarity_boost": 0.75, + "speaker_boost": false, + "stability": 0.5, + "voice_id": "pNInz6obpgDQGcFmaJgB" +} \ No newline at end of file diff --git a/agents/ten_packages/extension/fish_audio_tts/property.json b/agents/ten_packages/extension/fish_audio_tts/property.json index 9e26dfee..8053f9b4 100644 --- a/agents/ten_packages/extension/fish_audio_tts/property.json +++ b/agents/ten_packages/extension/fish_audio_tts/property.json @@ -1 +1,7 @@ -{} \ No newline at end of file +{ + "api_key": "${env:FISH_AUDIO_TTS_KEY}", + "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", + "optimize_streaming_latency": true, + "request_timeout_seconds": 30, + "base_url": "https://api.fish.audio" +} \ No newline at end of file diff --git a/playground/src/apis/routes.tsx b/playground/src/apis/routes.tsx deleted file mode 100644 index 2f1affc3..00000000 --- a/playground/src/apis/routes.tsx +++ /dev/null @@ -1,159 +0,0 @@ -import { LanguageMap } from '@/common/constant'; -import { NextRequest, NextResponse } from 'next/server'; - - -const { AGENT_SERVER_URL } = process.env; - -// Check if environment variables are available -if (!AGENT_SERVER_URL) { - throw "Environment variables AGENT_SERVER_URL are not available"; -} - - -export const voiceNameMap: LanguageMap = { - "zh-CN": { - azure: { - male: "zh-CN-YunxiNeural", - female: "zh-CN-XiaoxiaoNeural", - }, - elevenlabs: { - male: "pNInz6obpgDQGcFmaJgB", // Adam - female: "Xb7hH8MSUJpSbSDYk0k2", // Alice - }, - polly: { - male: "Zhiyu", - female: "Zhiyu", - }, - }, - "en-US": { - azure: { - male: "en-US-BrianNeural", - female: "en-US-AndrewMultilingualNeural", - }, - elevenlabs: { - male: "pNInz6obpgDQGcFmaJgB", // Adam - female: "Xb7hH8MSUJpSbSDYk0k2", // Alice - }, - polly: { - male: "Matthew", - female: "Ruth", - }, - }, - "ja-JP": { - azure: { - male: "ja-JP-KeitaNeural", - female: "ja-JP-NanamiNeural", - }, - }, - "ko-KR": { - azure: { - male: "ko-KR-InJoonNeural", - female: "ko-KR-JiMinNeural", - }, - }, -}; - -// Get the graph properties based on the graph name, language, and voice type -// This is the place where you can customize the properties for different graphs to override default property.json -export const getGraphProperties = (graphName: string, language: string, voiceType: string) => { - let localizationOptions = { - "greeting": "TEN agent connected. How can I help you today?", - "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", - } - - if (language === "zh-CN") { - localizationOptions = { - "greeting": "TEN Agent 已连接，需要我为您提供什么帮助?", - "checking_vision_text_items": "[\"让我看看你的摄像头...\",\"让我看一下...\",\"我看一下，请稍候...\"]", - } - } else if (language === "ja-JP") { - localizationOptions = { - "greeting": "TEN Agent エージェントに接続されました。今日は何をお手伝いしましょうか?", - "checking_vision_text_items": "[\"ちょっと見てみます...\",\"カメラをチェックします...\",\"少々お待ちください...\"]", - } - } else if (language === "ko-KR") { - localizationOptions = { - "greeting": "TEN Agent 에이전트에 연결되었습니다. 오늘은 무엇을 도와드릴까요?", - "checking_vision_text_items": "[\"조금만 기다려 주세요...\",\"카메라를 확인해 보겠습니다...\",\"잠시만 기다려 주세요...\"]", - } - } - - if (graphName == "camera_va_openai_azure") { - return { - "agora_rtc": { - "agora_asr_language": language, - }, - "openai_chatgpt": { - "model": "gpt-4o", - ...localizationOptions - }, - "azure_tts": { - "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] - } - } - } else if (graphName == "va_openai_azure") { - return { - "agora_rtc": { - "agora_asr_language": language, - }, - "openai_chatgpt": { - "model": "gpt-4o-mini", - ...localizationOptions - }, - "azure_tts": { - "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] - } - } - } else if (graphName == "va_qwen_rag") { - return { - "agora_rtc": { - "agora_asr_language": language, - }, - "azure_tts": { - "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] - } - } - } - return {} -} - -export async function startAgent(request: NextRequest) { - try { - const body = await request.json(); - const { - request_id, - channel_name, - user_uid, - graph_name, - language, - voice_type, - } = body; - - // Send a POST request to start the agent - const response = await fetch(`${AGENT_SERVER_URL}/start`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - request_id, - channel_name, - user_uid, - graph_name, - // Get the graph properties based on the graph name, language, and voice type - properties: getGraphProperties(graph_name, language, voice_type), - }), - }); - - const responseData = await response.json(); - - return NextResponse.json(responseData, { status: response.status }); - } catch (error) { - if (error instanceof Response) { - const errorData = await error.json(); - return NextResponse.json(errorData, { status: error.status }); - } else { - return NextResponse.json({ code: "1", data: null, msg: "Internal Server Error" }, { status: 500 }); - } - } -} diff --git a/playground/src/common/graph.ts b/playground/src/common/graph.ts new file mode 100644 index 00000000..6086abfd --- /dev/null +++ b/playground/src/common/graph.ts @@ -0,0 +1,490 @@ +import axios from "axios" +import { useCallback, useEffect, useState } from "react" +import { useAppDispatch, useAppSelector } from "./hooks" +import { + setAddonModules, + setGraph, + setGraphList, +} from "@/store/reducers/global" +import path from "path" + +export namespace AddonDef { + export type AttributeType = + | "string" + | "bool" + | "int32" + | "int64" + | "Uint32" + | "Uint64" + | "float64" + | "array" + | "buf" + + export type Attribute = { + type: AttributeType + } + + export type PropertyDefinition = { + name: string + attributes: Attribute + } + + export type Command = { + name: string + property?: PropertyDefinition[] + required?: string[] + result?: { + property: PropertyDefinition[] + required?: string[] + } + } + + export type ApiEndpoint = { + name: string + property?: PropertyDefinition[] + } + + export type Api = { + property?: Record + cmd_in?: Command[] + cmd_out?: Command[] + data_in?: ApiEndpoint[] + data_out?: ApiEndpoint[] + audio_frame_in?: ApiEndpoint[] + audio_frame_out?: ApiEndpoint[] + video_frame_in?: ApiEndpoint[] + video_frame_out?: ApiEndpoint[] + } + + export type Module = { + name: string + defaultProperty: Property + api: Api + } +} + +type Property = { + [key: string]: any +} + +type Node = { + name: string + addon: string + extensionGroup: string + app: string + property?: Property +} +type Command = { + name: string // Command name + dest: Array // Destination connections +} + +type Data = { + name: string // Data type name + dest: Array // Destination connections +} + +type AudioFrame = { + name: string // Audio frame type name + dest: Array // Destination connections +} + +type VideoFrame = { + name: string // Video frame type name + dest: Array // Destination connections +} + +type MsgConversion = { + type: string // Type of message conversion + rules: Array<{ + path: string // Path in the data structure + conversionMode: string // Mode of conversion (e.g., "replace", "append") + value?: string // Optional value for the conversion + originalPath?: string // Optional original path for mapping + }> + keepOriginal?: boolean // Whether to keep the original data +} + +type Destination = { + app: string // Application identifier + extensionGroup: string // Extension group name + extension: string // Extension name + msgConversion?: MsgConversion // Optional message conversion rules +} + +type Connection = { + app: string // Application identifier + extensionGroup: string // Extension group name + extension: string // Extension name + cmd?: Array // Optional command connections + data?: Array // Optional data connections + audioFrame?: Array // Optional audio frame connections + videoFrame?: Array // Optional video frame connections +} + +type Graph = { + id: string + autoStart: boolean + nodes: Node[] + connections: Connection[] +} + +const baseUrl = "/api/dev/v1" + +const useGraphManager = () => { + const dispatch = useAppDispatch() + const selectedGraphId = useAppSelector( + (state) => state.global.selectedGraphId, + ) + const selectedGraph = useAppSelector( + (state) => state.global.graphMap[selectedGraphId], + ) + + useEffect(() => { + if (selectedGraphId) { + fetchGraphDetails(selectedGraphId).then((graph) => { + dispatch(setGraph(graph)) + }) + } + }, [selectedGraphId]) + + const initializeGraphData = useCallback(async () => { + await reload() + const fetchedGraphs = await fetchGraphs() + const modules = await fetchInstalledAddons() + dispatch(setGraphList(fetchedGraphs.map((graph) => graph.id))) + dispatch(setAddonModules(modules)) + }, [baseUrl]) + + const getGraphModuleAddonValueByName = useCallback( + (nodeName: string) => { + if (!selectedGraph) { + return null + } + const node = selectedGraph.nodes.find((node) => node.name === nodeName) + if (!node) { + return null + } + return node + }, + [selectedGraph], + ) + + const fetchInstalledAddons = useCallback(async (): Promise< + AddonDef.Module[] + > => { + const response = await axios.get(`${baseUrl}/addons/extensions`) + const modules = response.data.data + const defaultProperties = await fetchAddonModuleProperties() + return modules.map((module: any) => ({ + name: module.name, + defaultProperty: defaultProperties[module.name], + api: module.api, + })) + }, [baseUrl]) + + const fetchGraphs = useCallback(async (): Promise => { + const response = await axios.get(`${baseUrl}/graphs`) + return response.data.data.map((graph: any) => ({ + id: graph.name, + autoStart: graph.auto_start, + nodes: [], + connections: [], + })) + }, [baseUrl]) + + const fetchGraphDetails = useCallback( + async (graphId: string): Promise => { + const nodesResponse = await axios.get( + `${baseUrl}/graphs/${graphId}/nodes`, + ) + const connectionsResponse = await axios.get( + `${baseUrl}/graphs/${graphId}/connections`, + ) + + // Map nodes to their refined structure + const nodes: Node[] = nodesResponse.data.data.map((node: any) => ({ + name: node.name, + addon: node.addon, + extensionGroup: node.extension_group, + app: node.app, + property: node.property || {}, + })) + + // Map connections to the refined structure + const connections: Connection[] = connectionsResponse.data.data.map( + (connection: any) => ({ + app: connection.app, + extensionGroup: connection.extension_group, + extension: connection.extension, + cmd: connection.cmd?.map((cmd: any) => ({ + name: cmd.name, + dest: cmd.dest.map((dest: any) => ({ + app: dest.app, + extensionGroup: dest.extension_group, + extension: dest.extension, + msgConversion: dest.msgConversion + ? { + type: dest.msgConversion.type, + rules: dest.msgConversion.rules.map((rule: any) => ({ + path: rule.path, + conversionMode: rule.conversionMode, + value: rule.value, + originalPath: rule.originalPath, + })), + keepOriginal: dest.msgConversion.keepOriginal, + } + : undefined, + })), + })), + data: connection.data?.map((data: any) => ({ + name: data.name, + dest: data.dest.map((dest: any) => ({ + app: dest.app, + extensionGroup: dest.extension_group, + extension: dest.extension, + msgConversion: dest.msgConversion + ? { + type: dest.msgConversion.type, + rules: dest.msgConversion.rules.map((rule: any) => ({ + path: rule.path, + conversionMode: rule.conversionMode, + value: rule.value, + originalPath: rule.originalPath, + })), + keepOriginal: dest.msgConversion.keepOriginal, + } + : undefined, + })), + })), + audioFrame: connection.audio_frame?.map((audioFrame: any) => ({ + name: audioFrame.name, + dest: audioFrame.dest.map((dest: any) => ({ + app: dest.app, + extensionGroup: dest.extension_group, + extension: dest.extension, + msgConversion: dest.msgConversion + ? { + type: dest.msgConversion.type, + rules: dest.msgConversion.rules.map((rule: any) => ({ + path: rule.path, + conversionMode: rule.conversionMode, + value: rule.value, + originalPath: rule.originalPath, + })), + keepOriginal: dest.msgConversion.keepOriginal, + } + : undefined, + })), + })), + videoFrame: connection.videoFrame?.map((videoFrame: any) => ({ + name: videoFrame.name, + dest: videoFrame.dest.map((dest: any) => ({ + app: dest.app, + extensionGroup: dest.extension_group, + extension: dest.extension, + msgConversion: dest.msgConversion + ? { + type: dest.msgConversion.type, + rules: dest.msgConversion.rules.map((rule: any) => ({ + path: rule.path, + conversionMode: rule.conversionMode, + value: rule.value, + originalPath: rule.originalPath, + })), + keepOriginal: dest.msgConversion.keepOriginal, + } + : undefined, + })), + })), + }), + ) + + return { + id: graphId, + autoStart: true, + nodes, + connections, + } + }, + [baseUrl], + ) + + const updateGraph = useCallback( + async (graphId: string, updates: Partial): Promise => { + const { autoStart, nodes, connections } = updates + const payload: any = {} + + // Map autoStart field + if (autoStart !== undefined) payload.auto_start = autoStart + + // Map nodes to the payload + if (nodes) { + payload.nodes = nodes.map((node) => ({ + name: node.name, + addon: node.addon, + extension_group: node.extensionGroup, + app: node.app, + property: node.property, + })) + } + + // Map connections to the payload + if (connections) { + payload.connections = connections.map((connection) => ({ + app: connection.app, + extension_group: connection.extensionGroup, + extension: connection.extension, + cmd: connection.cmd?.map((cmd) => ({ + name: cmd.name, + dest: cmd.dest.map((dest) => ({ + app: dest.app, + extension_group: dest.extensionGroup, + extension: dest.extension, + msgConversion: dest.msgConversion + ? { + type: dest.msgConversion.type, + rules: dest.msgConversion.rules.map((rule) => ({ + path: rule.path, + conversionMode: rule.conversionMode, + value: rule.value, + originalPath: rule.originalPath, + })), + keepOriginal: dest.msgConversion.keepOriginal, + } + : undefined, + })), + })), + data: connection.data?.map((data) => ({ + name: data.name, + dest: data.dest.map((dest) => ({ + app: dest.app, + extension_group: dest.extensionGroup, + extension: dest.extension, + msgConversion: dest.msgConversion + ? { + type: dest.msgConversion.type, + rules: dest.msgConversion.rules.map((rule) => ({ + path: rule.path, + conversionMode: rule.conversionMode, + value: rule.value, + originalPath: rule.originalPath, + })), + keepOriginal: dest.msgConversion.keepOriginal, + } + : undefined, + })), + })), + audio_frame: connection.audioFrame?.map((audioFrame) => ({ + name: audioFrame.name, + dest: audioFrame.dest.map((dest) => ({ + app: dest.app, + extension_group: dest.extensionGroup, + extension: dest.extension, + msgConversion: dest.msgConversion + ? { + type: dest.msgConversion.type, + rules: dest.msgConversion.rules.map((rule) => ({ + path: rule.path, + conversionMode: rule.conversionMode, + value: rule.value, + originalPath: rule.originalPath, + })), + keepOriginal: dest.msgConversion.keepOriginal, + } + : undefined, + })), + })), + video_frame: connection.videoFrame?.map((videoFrame) => ({ + name: videoFrame.name, + dest: videoFrame.dest.map((dest) => ({ + app: dest.app, + extension_group: dest.extensionGroup, + extension: dest.extension, + msgConversion: dest.msgConversion + ? { + type: dest.msgConversion.type, + rules: dest.msgConversion.rules.map((rule) => ({ + path: rule.path, + conversionMode: rule.conversionMode, + value: rule.value, + originalPath: rule.originalPath, + })), + keepOriginal: dest.msgConversion.keepOriginal, + } + : undefined, + })), + })), + })) + } + + // Send updated data to the server + await axios.put(`${baseUrl}/graphs/${graphId}`, payload) + + // Save additional properties if needed + await saveProperty() + + // Update the local state with the latest graph details + const updatedGraph = await fetchGraphDetails(graphId) + dispatch(setGraph(updatedGraph)) + }, + [baseUrl], + ) + + const getCompatibleMessages = useCallback( + async (payload: { + app: string + graph: string + extensionGroup: string + extension: string + msgType: string + msgDirection: string + msgName: string + }): Promise => { + const response = await axios.post( + `${baseUrl}/messages/compatible`, + payload, + ) + return response.data + }, + [baseUrl], + ) + + const saveProperty = useCallback(async (): Promise => { + await axios.put(`${baseUrl}/property`) + }, [baseUrl]) + + const fetchAddonModuleProperties = useCallback(async (): Promise< + Record> + > => { + const response = await axios.get(`${baseUrl}/addons/default-properties`) + const properties = response.data.data + const result: Record> = {} + for (const property of properties) { + result[property.addon] = property.property + } + return result + }, [baseUrl]) + + const reload = useCallback(async (): Promise => { + const response = await axios.post(`${baseUrl}/packages/reload`) + return response.data + }, [baseUrl]) + + return { + initializeGraphData, + fetchInstalledAddons, + fetchGraphs, + fetchGraphDetails, + updateGraph, + getCompatibleMessages, + saveProperty, + fetchAddonModuleProperties, + reload, + getModuleAddonValueByName: getGraphModuleAddonValueByName, + selectedGraph, + } +} + +export { useGraphManager } +export type { Graph, Node, Connection, Command } diff --git a/playground/src/common/hooks.ts b/playground/src/common/hooks.ts index 41e031f3..76cc9636 100644 --- a/playground/src/common/hooks.ts +++ b/playground/src/common/hooks.ts @@ -129,7 +129,7 @@ export const usePrevious = (value: any) => { }; export const useGraphExtensions = () => { - const graphName = useAppSelector((state) => state.global.graphName); + const graphName = useAppSelector((state) => state.global.selectedGraphId); const nodes = useAppSelector((state) => state.global.extensions); const overridenProperties = useAppSelector( (state) => state.global.overridenProperties @@ -168,4 +168,9 @@ export const useGraphExtensions = () => { export const useExtensionsMetadataNames = (): string[] => { const metadata = useAppSelector((state) => state.global.extensionMetadata); return Object.keys(metadata) -}; \ No newline at end of file +}; + +export const useGraph = (graphId: string) => { + const graph = useAppSelector((state) => state.global.graphMap[graphId]); + return graph; +} \ No newline at end of file diff --git a/playground/src/common/request.ts b/playground/src/common/request.ts index 95c2c176..c40f2a55 100644 --- a/playground/src/common/request.ts +++ b/playground/src/common/request.ts @@ -98,37 +98,4 @@ export const apiPing = async (channel: string) => { let resp: any = await axios.post(url, data) resp = (resp.data) || {} return resp -} - -export const apiGetGraphs = async () => { - // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL - const url = `/api/dev/v1/graphs` - let resp: any = await axios.get(url) - resp = (resp.data) || {} - return resp -} - -export const apiGetExtensionMetadata = async () => { - // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL - const url = `/api/dev/v1/addons/extensions` - let resp: any = await axios.get(url) - resp = (resp.data) || {} - return resp -} - -export const apiGetNodes = async (graphName: string) => { - // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL - const url = `/api/dev/v1/graphs/${graphName}/nodes` - let resp: any = await axios.get(url) - resp = (resp.data) || {} - return resp -} - - -export const apiReloadGraph = async (): Promise => { - // look at app/apis/route.tsx for the server-side implementation - const url = `/api/dev/v1/packages/reload` - let resp: any = await axios.post(url) - resp = (resp.data) || {} - return resp } \ No newline at end of file diff --git a/playground/src/components/Chat/ChatCard.tsx b/playground/src/components/Chat/ChatCard.tsx index 16392342..cab36b1c 100644 --- a/playground/src/components/Chat/ChatCard.tsx +++ b/playground/src/components/Chat/ChatCard.tsx @@ -16,18 +16,13 @@ import { useAppSelector, GRAPH_OPTIONS, isRagGraph, - apiGetGraphs, - apiGetNodes, useGraphExtensions, - apiGetExtensionMetadata, - apiReloadGraph, } from "@/common"; import { setRtmConnected, addChatItem, setExtensionMetadata, - setGraphName, - setGraphs, + setSelectedGraphId, setLanguage, setExtensions, setOverridenPropertiesByGraph, @@ -46,9 +41,9 @@ export default function ChatCard(props: { className?: string }) { const rtmConnected = useAppSelector((state) => state.global.rtmConnected); const dispatch = useAppDispatch(); - const graphs = useAppSelector((state) => state.global.graphs); + const graphs = useAppSelector((state) => state.global.graphList); const extensions = useAppSelector((state) => state.global.extensions); - const graphName = useAppSelector((state) => state.global.graphName); + const graphName = useAppSelector((state) => state.global.selectedGraphId); const chatItems = useAppSelector((state) => state.global.chatItems); const agentConnected = useAppSelector((state) => state.global.agentConnected); const graphExtensions = useGraphExtensions(); @@ -81,44 +76,8 @@ export default function ChatCard(props: { className?: string }) { // const chatItems = genRandomChatList(10) const chatRef = React.useRef(null); - React.useEffect(() => { - apiReloadGraph().then(() => { - Promise.all([apiGetGraphs(), apiGetExtensionMetadata()]).then( - (res: any) => { - let [graphRes, metadataRes] = res; - let graphs = graphRes["data"].map((item: any) => item["name"]); - - let metadata = metadataRes["data"]; - let metadataMap: Record = {}; - metadata.forEach((item: any) => { - metadataMap[item["name"]] = item; - }); - dispatch(setGraphs(graphs)); - dispatch(setExtensionMetadata(metadataMap)); - } - ); - }); - }, []); - - React.useEffect(() => { - if (graphName !== "" && !extensions[graphName]) { - apiGetNodes(graphName).then((res: any) => { - let nodes = res["data"]; - let nodesMap: Record = {}; - nodes.forEach((item: any) => { - nodesMap[item["name"]] = item; - }); - dispatch(setExtensions({ graphName, nodesMap })); - }); - } - }, [graphName]); - useAutoScroll(chatRef); - const onGraphNameChange = (val: any) => { - dispatch(setGraphName(val)); - }; - const onTextChanged = (text: IRTMTextItem) => { console.log("[rtm] onTextChanged", text); if (text.type == ERTMTextType.TRANSCRIBE) { diff --git a/playground/src/components/Chat/ChatCfgSelect.tsx b/playground/src/components/Chat/ChatCfgSelect.tsx index ef8c38a6..6b63941a 100644 --- a/playground/src/components/Chat/ChatCfgSelect.tsx +++ b/playground/src/components/Chat/ChatCfgSelect.tsx @@ -43,11 +43,13 @@ import { GRAPH_OPTIONS, useGraphExtensions, useExtensionsMetadataNames, + useGraph, } from "@/common" import type { Language } from "@/types" import { - setGraphName, + setSelectedGraphId, setLanguage, + setOverridenAddonsByGraph, setOverridenPropertiesByGraph, } from "@/store/reducers/global" import { cn } from "@/lib/utils" @@ -56,15 +58,16 @@ import { zodResolver } from "@hookform/resolvers/zod" import { useForm } from "react-hook-form" import { z } from "zod" import { toast } from "sonner" +import { Graph, useGraphManager } from "@/common/graph" export function RemoteGraphSelect() { const dispatch = useAppDispatch() - const graphName = useAppSelector((state) => state.global.graphName) - const graphs = useAppSelector((state) => state.global.graphs) + const graphName = useAppSelector((state) => state.global.selectedGraphId) + const graphs = useAppSelector((state) => state.global.graphList) const agentConnected = useAppSelector((state) => state.global.agentConnected) const onGraphNameChange = (val: string) => { - dispatch(setGraphName(val)) + dispatch(setSelectedGraphId(val)) } const graphOptions = graphs.map((item) => ({ @@ -95,47 +98,39 @@ export function RemoteGraphSelect() { } export function RemoteModuleCfgSheet() { - const dispatch = useAppDispatch() - const graphExtensions = useGraphExtensions() - const graphName = useAppSelector((state) => state.global.graphName) - const extensionMetadata = useAppSelector( - (state) => state.global.extensionMetadata, - ) - const extensionMetadataNames = useExtensionsMetadataNames() - const overridenProperties = useAppSelector( - (state) => state.global.overridenProperties, - ) + const addonModules = useAppSelector((state) => state.global.addonModules) + const {getModuleAddonValueByName, selectedGraph, updateGraph} = useGraphManager() + + const modules = React.useMemo(() => { + let result: { stt: string[], llm: string[], tts: string[] } = { + stt: [], + llm: [], + tts: [], + } + + addonModules.forEach((module) => { + if (module.name.includes("stt") || module.name.includes("asr")) { + result.stt.push(module.name) + } else if (module.name.includes("llm")) { + result.llm.push(module.name) + } else if (module.name.includes("tts")) { + result.tts.push(module.name) + } + }) + return result + }, [addonModules]) - const sttExtensionNames = React.useMemo(() => { - return extensionMetadataNames.filter((item) => - item.includes("stt") || item.includes("asr"), - ) - }, [extensionMetadata]) - - const llmExtensionNames = React.useMemo(() => { - return extensionMetadataNames.filter((item) => - item.includes("llm"), - ) - }, [extensionMetadata]) - - const ttsExtensionNames = React.useMemo(() => { - return extensionMetadataNames.filter((item) => - item.includes("tts"), - ) - }, [extensionMetadata]) - - console.log(extensionMetadataNames) const metadata = { - STT: { type: "string", options: sttExtensionNames }, - LLM: { type: "string", options: llmExtensionNames }, - TTS: { type: "string", options: ttsExtensionNames }, + stt: { type: "string", options: modules.stt }, + llm: { type: "string", options: modules.llm }, + tts: { type: "string", options: modules.tts }, } const initialData = { - STT: null, - LLM: null, - TTS: null, + stt: getModuleAddonValueByName("stt")?.addon, + llm: getModuleAddonValueByName("llm")?.addon, + tts: getModuleAddonValueByName("tts")?.addon, } return ( @@ -157,12 +152,44 @@ export function RemoteModuleCfgSheet() { won't modify the property.json file. - +
- {}} + onUpdate={async (data) => { + // clone the overridenAddons + const selectedGraphCopy:Graph = JSON.parse(JSON.stringify(selectedGraph)) + const nodes = selectedGraphCopy?.nodes || [] + let needUpdate = false + for (const node of nodes) { + if (data.stt && node.name === "stt" && node.addon !== data.stt) { + node.addon = data.stt + node.property = addonModules.find((module) => module.name === data.stt)?.defaultProperty + needUpdate = true + } + if (data.llm && node.name === "lln" && node.addon !== data.llm) { + node.addon = data.llm + node.property = addonModules.find((module) => module.name === data.llm)?.defaultProperty + needUpdate = true + } + if (data.tts && node.name === "tts" && node.addon !== data.tts) { + node.addon = data.tts + node.property = addonModules.find((module) => module.name === data.tts)?.defaultProperty + needUpdate = true + } + } + if (needUpdate) { + try { + await updateGraph(selectedGraphCopy.id, selectedGraphCopy) + toast.success("Modules updated", { + description: `Graph: ${selectedGraphCopy.id}`, + }) + } catch (e) { + toast.error("Failed to update modules") + } + } + }} />
@@ -179,7 +206,7 @@ export function RemoteModuleCfgSheet() { export function RemotePropertyCfgSheet() { const dispatch = useAppDispatch() const graphExtensions = useGraphExtensions() - const graphName = useAppSelector((state) => state.global.graphName) + const graphName = useAppSelector((state) => state.global.selectedGraphId) const extensionMetadata = useAppSelector( (state) => state.global.extensionMetadata, ) @@ -294,7 +321,7 @@ const GraphModuleCfgForm = ({ metadata, onUpdate, }: { - initialData: Record + initialData: Record metadata: Record onUpdate: (data: Record) => void }) => { @@ -313,6 +340,7 @@ const GraphModuleCfgForm = ({ onUpdate(data) } + return (
diff --git a/playground/src/components/Layout/Action.tsx b/playground/src/components/Layout/Action.tsx index 9405f232..1d87dc4e 100644 --- a/playground/src/components/Layout/Action.tsx +++ b/playground/src/components/Layout/Action.tsx @@ -27,7 +27,7 @@ export default function Action(props: { className?: string }) { const userId = useAppSelector((state) => state.global.options.userId); const language = useAppSelector((state) => state.global.language); const voiceType = useAppSelector((state) => state.global.voiceType); - const graphName = useAppSelector((state) => state.global.graphName); + const graphName = useAppSelector((state) => state.global.selectedGraphId); const overridenProperties = useAppSelector( (state) => state.global.overridenProperties ); diff --git a/playground/src/components/authInitializer/index.tsx b/playground/src/components/authInitializer/index.tsx index c19e9fb8..4f51b9fd 100644 --- a/playground/src/components/authInitializer/index.tsx +++ b/playground/src/components/authInitializer/index.tsx @@ -3,6 +3,7 @@ import { ReactNode, useEffect } from "react" import { useAppDispatch, getOptionsFromLocal, getRandomChannel, getRandomUserId, getOverridenPropertiesFromLocal } from "@/common" import { setOptions, reset, setOverridenProperties } from "@/store/reducers/global" +import { useGraphManager } from "@/common/graph"; interface AuthInitializerProps { children: ReactNode; @@ -11,10 +12,12 @@ interface AuthInitializerProps { const AuthInitializer = (props: AuthInitializerProps) => { const { children } = props; const dispatch = useAppDispatch() + const {initializeGraphData} = useGraphManager() useEffect(() => { if (typeof window !== "undefined") { const options = getOptionsFromLocal() + initializeGraphData() const overridenProperties = getOverridenPropertiesFromLocal() if (options && options.channel) { dispatch(reset()) diff --git a/playground/src/middleware.tsx b/playground/src/middleware.tsx index f3577ff3..48d74439 100644 --- a/playground/src/middleware.tsx +++ b/playground/src/middleware.tsx @@ -1,7 +1,5 @@ // middleware.js import { NextRequest, NextResponse } from 'next/server'; -import { startAgent } from './apis/routes'; - const { AGENT_SERVER_URL, TEN_DEV_SERVER_URL } = process.env; @@ -14,20 +12,28 @@ if (!TEN_DEV_SERVER_URL) { throw "Environment variables TEN_DEV_SERVER_URL are not available"; } -export function middleware(req: NextRequest) { +export async function middleware(req: NextRequest) { const { pathname } = req.nextUrl; const url = req.nextUrl.clone(); + if (pathname.startsWith(`/api/agents/`)) { - if (!pathname.startsWith('/api/agents/start')) { - // Proxy all other agents API requests - url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/agents/', '/')}`; + // if (!pathname.startsWith('/api/agents/start')) { + // Proxy all other agents API requests + url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/agents/', '/')}`; - // console.log(`Rewriting request to ${url.href}`); - return NextResponse.rewrite(url); - } else { - return NextResponse.next(); + try { + const body = await req.json(); + console.log(`Request to ${pathname} with body ${JSON.stringify(body)}`); + } catch (e) { + console.log(`Request to ${pathname} ${e}`); } + + // console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + // } else { + // return NextResponse.next(); + // } } else if (pathname.startsWith(`/api/vector/`)) { // Proxy all other documents requests @@ -43,16 +49,12 @@ export function middleware(req: NextRequest) { return NextResponse.rewrite(url); } else if (pathname.startsWith('/api/dev/')) { - // Proxy all other documents requests - const url = req.nextUrl.clone(); - url.href = `${TEN_DEV_SERVER_URL}${pathname.replace('/api/dev/', '/api/dev-server/')}`; - - // console.log(`Rewriting request to ${url.href}`); - return NextResponse.rewrite(url); - } else if (pathname.startsWith('/api/dev/')) { + if (pathname.startsWith('/api/dev/v1/addons/default-properties')) { + url.href = `${AGENT_SERVER_URL}/dev-tmp/addons/default-properties`; + console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + } - // Proxy all other documents requests - const url = req.nextUrl.clone(); url.href = `${TEN_DEV_SERVER_URL}${pathname.replace('/api/dev/', '/api/dev-server/')}`; // console.log(`Rewriting request to ${url.href}`); diff --git a/playground/src/store/reducers/global.ts b/playground/src/store/reducers/global.ts index 112128a3..177bcee5 100644 --- a/playground/src/store/reducers/global.ts +++ b/playground/src/store/reducers/global.ts @@ -15,6 +15,8 @@ import { setOverridenPropertiesToLocal, deepMerge, } from "@/common"; +import { AddonDef, Graph } from "@/common/graph"; +import { set } from "react-hook-form"; export interface InitialState { options: IOptions; @@ -25,11 +27,13 @@ export interface InitialState { language: Language; voiceType: VoiceType; chatItems: IChatItem[]; - graphName: string; - graphs: string[]; + selectedGraphId: string; + graphList: string[]; extensions: Record; overridenProperties: Record; extensionMetadata: Record; + graphMap: Record; + addonModules: AddonDef.Module[]; // addon modules mobileActiveTab: EMobileActiveTab; } @@ -43,11 +47,13 @@ const getInitialState = (): InitialState => { language: "en-US", voiceType: "male", chatItems: [], - graphName: "", - graphs: [], + selectedGraphId: "", + graphList: [], extensions: {}, overridenProperties: {}, extensionMetadata: {}, + graphMap: {}, + addonModules: [], mobileActiveTab: EMobileActiveTab.AGENT, }; }; @@ -136,11 +142,11 @@ export const globalSlice = createSlice({ setLanguage: (state, action: PayloadAction) => { state.language = action.payload; }, - setGraphName: (state, action: PayloadAction) => { - state.graphName = action.payload; + setSelectedGraphId: (state, action: PayloadAction) => { + state.selectedGraphId = action.payload; }, - setGraphs: (state, action: PayloadAction) => { - state.graphs = action.payload; + setGraphList: (state, action: PayloadAction) => { + state.graphList = action.payload; }, setExtensions: (state, action: PayloadAction>) => { let { graphName, nodesMap } = action.payload; @@ -183,6 +189,14 @@ export const globalSlice = createSlice({ COLOR_LIST[0].active ); }, + setGraph: (state, action: PayloadAction) => { + let graphMap = JSON.parse(JSON.stringify(state.graphMap)); + graphMap[action.payload.id] = action.payload; + state.graphMap = graphMap; + }, + setAddonModules: (state, action: PayloadAction[]>) => { + state.addonModules = JSON.parse(JSON.stringify(action.payload)); + } }, }); @@ -196,13 +210,15 @@ export const { addChatItem, setThemeColor, setLanguage, - setGraphName, - setGraphs, + setSelectedGraphId, + setGraphList, setExtensions, setExtensionMetadata, setOverridenProperties, setOverridenPropertiesByGraph, setMobileActiveTab, + setGraph, + setAddonModules, } = globalSlice.actions; export default globalSlice.reducer; diff --git a/server/internal/code.go b/server/internal/code.go index 8a4b61b2..d518f821 100644 --- a/server/internal/code.go +++ b/server/internal/code.go @@ -23,6 +23,8 @@ var ( codeErrStopWorkerFailed = NewCode("10102", "stop worker failed") codeErrHttpStatusNotOk = NewCode("10103", "http status not 200") codeErrUpdateWorkerFailed = NewCode("10104", "update worker failed") + codeErrReadDirectoryFailed = NewCode("10105", "read directory failed") + codeErrReadFileFailed = NewCode("10106", "read file failed") ) func NewCode(code string, msg string) *Code { diff --git a/server/internal/http_server.go b/server/internal/http_server.go index cc9a29fc..8943d555 100644 --- a/server/internal/http_server.go +++ b/server/internal/http_server.go @@ -54,7 +54,6 @@ type StartReq struct { Token string `json:"token,omitempty"` WorkerHttpServerPort int32 `json:"worker_http_server_port,omitempty"` Properties map[string]map[string]interface{} `json:"properties,omitempty"` - Addons map[string]interface{} `json:"addons,omitempty"` QuitTimeoutSeconds int `json:"timeout,omitempty"` } @@ -109,6 +108,47 @@ func (s *HttpServer) handlerList(c *gin.Context) { s.output(c, codeSuccess, filtered) } +func (s *HttpServer) handleAddonDefaultProperties(c *gin.Context) { + // Get the base directory path + baseDir := "./agents/ten_packages/extension" + + // Read all folders under the base directory + entries, err := os.ReadDir(baseDir) + if err != nil { + slog.Error("failed to read extension directory", "err", err, logTag) + s.output(c, codeErrReadDirectoryFailed, http.StatusInternalServerError) + return + } + + // Iterate through each folder and read the property.json file + var addons []map[string]interface{} + for _, entry := range entries { + if entry.IsDir() { + addonName := entry.Name() + propertyFilePath := fmt.Sprintf("%s/%s/property.json", baseDir, addonName) + content, err := os.ReadFile(propertyFilePath) + if err != nil { + slog.Warn("failed to read property file", "addon", addonName, "err", err, logTag) + continue + } + + var properties map[string]interface{} + err = json.Unmarshal(content, &properties) + if err != nil { + slog.Warn("failed to parse property file", "addon", addonName, "err", err, logTag) + continue + } + + addons = append(addons, map[string]interface{}{ + "addon": addonName, + "property": properties, + }) + } + } + + s.output(c, codeSuccess, addons) +} + func (s *HttpServer) handlerPing(c *gin.Context) { var req PingReq @@ -146,6 +186,7 @@ func (s *HttpServer) handlerStart(c *gin.Context) { slog.Info("handlerStart start", "workersRunning", workersRunning, logTag) var req StartReq + if err := c.ShouldBindBodyWith(&req, binding.JSON); err != nil { slog.Error("handlerStart params invalid", "err", err, "requestId", req.RequestId, logTag) s.output(c, codeErrParamsInvalid, http.StatusBadRequest) @@ -450,20 +491,6 @@ func (s *HttpServer) processProperty(req *StartReq) (propertyJsonFile string, lo graphMap["auto_start"] = true } - // Process Addons property - for addonKey, addonValue := range req.Addons { - for _, graph := range newGraphs { - graphMap, _ := graph.(map[string]interface{}) - nodes, _ := graphMap["nodes"].([]interface{}) - for _, node := range nodes { - nodeMap, _ := node.(map[string]interface{}) - if nodeMap["name"] == addonKey { - nodeMap["addon"] = addonValue - } - } - } - } - // Set additional properties to property.json for extensionName, props := range req.Properties { if extensionName != "" { @@ -530,6 +557,7 @@ func (s *HttpServer) Start() { r.POST("/start", s.handlerStart) r.POST("/stop", s.handlerStop) r.POST("/ping", s.handlerPing) + r.GET("/dev-tmp/addons/default-properties", s.handleAddonDefaultProperties) r.POST("/token/generate", s.handlerGenerateToken) r.GET("/vector/document/preset/list", s.handlerVectorDocumentPresetList) r.POST("/vector/document/update", s.handlerVectorDocumentUpdate) From 6b99eb535b3377f0b8b11d159f86f80d8bff3aad Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Fri, 29 Nov 2024 23:20:10 +0800 Subject: [PATCH 07/28] feat: first draft done for property editing --- playground/src/common/graph.ts | 7 +- playground/src/common/hooks.ts | 48 +--- playground/src/common/storage.ts | 18 +- playground/src/components/Chat/ChatCard.tsx | 2 - .../src/components/Chat/ChatCfgSelect.tsx | 255 +++++++++++++----- .../src/components/authInitializer/index.tsx | 6 +- playground/src/store/reducers/global.ts | 38 --- 7 files changed, 196 insertions(+), 178 deletions(-) diff --git a/playground/src/common/graph.ts b/playground/src/common/graph.ts index 6086abfd..9096a160 100644 --- a/playground/src/common/graph.ts +++ b/playground/src/common/graph.ts @@ -7,6 +7,7 @@ import { setGraphList, } from "@/store/reducers/global" import path from "path" +import { deepMerge } from "./utils" export namespace AddonDef { export type AttributeType = @@ -136,9 +137,11 @@ const useGraphManager = () => { const selectedGraphId = useAppSelector( (state) => state.global.selectedGraphId, ) - const selectedGraph = useAppSelector( - (state) => state.global.graphMap[selectedGraphId], + const graphMap = useAppSelector( + (state) => state.global.graphMap, ) + const selectedGraph = graphMap[selectedGraphId] + const addonModules = useAppSelector((state) => state.global.addonModules) useEffect(() => { if (selectedGraphId) { diff --git a/playground/src/common/hooks.ts b/playground/src/common/hooks.ts index 76cc9636..e28d286c 100644 --- a/playground/src/common/hooks.ts +++ b/playground/src/common/hooks.ts @@ -5,6 +5,7 @@ import { deepMerge, normalizeFrequencies } from "./utils"; import { useState, useEffect, useMemo, useRef } from "react"; import type { AppDispatch, AppStore, RootState } from "../store"; import { useDispatch, useSelector, useStore } from "react-redux"; +import { Node, AddonDef } from "./graph"; // import { Grid } from "antd" // const { useBreakpoint } = Grid; @@ -127,50 +128,3 @@ export const usePrevious = (value: any) => { return ref.current; }; - -export const useGraphExtensions = () => { - const graphName = useAppSelector((state) => state.global.selectedGraphId); - const nodes = useAppSelector((state) => state.global.extensions); - const overridenProperties = useAppSelector( - (state) => state.global.overridenProperties - ); - const [graphExtensions, setGraphExtensions] = useState>( - {} - ); - - useEffect(() => { - if (nodes && nodes[graphName]) { - let extensions: Record = {}; - let extensionsByGraph = JSON.parse(JSON.stringify(nodes[graphName])); - let overriden = overridenProperties[graphName] || {}; - for (const key of Object.keys(extensionsByGraph)) { - if (!overriden[key]) { - extensions[key] = extensionsByGraph[key]; - continue; - } - extensions[key] = { - addon: extensionsByGraph[key].addon, - name: extensionsByGraph[key].name, - }; - extensions[key].property = deepMerge( - extensionsByGraph[key].property, - overriden[key] - ); - } - setGraphExtensions(extensions); - } - }, [graphName, nodes, overridenProperties]); - - return graphExtensions; -}; - - -export const useExtensionsMetadataNames = (): string[] => { - const metadata = useAppSelector((state) => state.global.extensionMetadata); - return Object.keys(metadata) -}; - -export const useGraph = (graphId: string) => { - const graph = useAppSelector((state) => state.global.graphMap[graphId]); - return graph; -} \ No newline at end of file diff --git a/playground/src/common/storage.ts b/playground/src/common/storage.ts index 54c956c6..e9dd9930 100644 --- a/playground/src/common/storage.ts +++ b/playground/src/common/storage.ts @@ -11,24 +11,8 @@ export const getOptionsFromLocal = () => { return DEFAULT_OPTIONS } -export const getOverridenPropertiesFromLocal = () => { - if (typeof window !== "undefined") { - const data = localStorage.getItem(OVERRIDEN_PROPERTIES_KEY) - if (data) { - return JSON.parse(data) - } - } - return {} -} - export const setOptionsToLocal = (options: IOptions) => { if (typeof window !== "undefined") { localStorage.setItem(OPTIONS_KEY, JSON.stringify(options)) } -} - -export const setOverridenPropertiesToLocal = (properties: Record) => { - if (typeof window !== "undefined") { - localStorage.setItem(OVERRIDEN_PROPERTIES_KEY, JSON.stringify(properties)) - } -} +} \ No newline at end of file diff --git a/playground/src/components/Chat/ChatCard.tsx b/playground/src/components/Chat/ChatCard.tsx index cab36b1c..97f10580 100644 --- a/playground/src/components/Chat/ChatCard.tsx +++ b/playground/src/components/Chat/ChatCard.tsx @@ -16,7 +16,6 @@ import { useAppSelector, GRAPH_OPTIONS, isRagGraph, - useGraphExtensions, } from "@/common"; import { setRtmConnected, @@ -46,7 +45,6 @@ export default function ChatCard(props: { className?: string }) { const graphName = useAppSelector((state) => state.global.selectedGraphId); const chatItems = useAppSelector((state) => state.global.chatItems); const agentConnected = useAppSelector((state) => state.global.agentConnected); - const graphExtensions = useGraphExtensions(); const extensionMetadata = useAppSelector( (state) => state.global.extensionMetadata ); diff --git a/playground/src/components/Chat/ChatCfgSelect.tsx b/playground/src/components/Chat/ChatCfgSelect.tsx index 6b63941a..a9b6192e 100644 --- a/playground/src/components/Chat/ChatCfgSelect.tsx +++ b/playground/src/components/Chat/ChatCfgSelect.tsx @@ -41,24 +41,20 @@ import { LANGUAGE_OPTIONS, useAppSelector, GRAPH_OPTIONS, - useGraphExtensions, - useExtensionsMetadataNames, - useGraph, } from "@/common" import type { Language } from "@/types" import { setSelectedGraphId, setLanguage, - setOverridenAddonsByGraph, - setOverridenPropertiesByGraph, } from "@/store/reducers/global" import { cn } from "@/lib/utils" -import { SettingsIcon, LoaderCircleIcon, BoxesIcon } from "lucide-react" +import { SettingsIcon, LoaderCircleIcon, BoxesIcon, Trash2Icon } from "lucide-react" import { zodResolver } from "@hookform/resolvers/zod" import { useForm } from "react-hook-form" import { z } from "zod" import { toast } from "sonner" -import { Graph, useGraphManager } from "@/common/graph" +import { AddonDef, Graph, useGraphManager } from "@/common/graph" +import { Popover, PopoverContent, PopoverTrigger } from "../ui/popover" export function RemoteGraphSelect() { const dispatch = useAppDispatch() @@ -99,7 +95,7 @@ export function RemoteGraphSelect() { export function RemoteModuleCfgSheet() { const addonModules = useAppSelector((state) => state.global.addonModules) - const {getModuleAddonValueByName, selectedGraph, updateGraph} = useGraphManager() + const { getModuleAddonValueByName, selectedGraph, updateGraph } = useGraphManager() const modules = React.useMemo(() => { let result: { stt: string[], llm: string[], tts: string[] } = { @@ -157,9 +153,9 @@ export function RemoteModuleCfgSheet() { { + onUpdate={async (data) => { // clone the overridenAddons - const selectedGraphCopy:Graph = JSON.parse(JSON.stringify(selectedGraph)) + const selectedGraphCopy: Graph = JSON.parse(JSON.stringify(selectedGraph)) const nodes = selectedGraphCopy?.nodes || [] let needUpdate = false for (const node of nodes) { @@ -205,16 +201,15 @@ export function RemoteModuleCfgSheet() { export function RemotePropertyCfgSheet() { const dispatch = useAppDispatch() - const graphExtensions = useGraphExtensions() + const {selectedGraph, updateGraph} = useGraphManager() const graphName = useAppSelector((state) => state.global.selectedGraphId) - const extensionMetadata = useAppSelector( - (state) => state.global.extensionMetadata, - ) - const overridenProperties = useAppSelector( - (state) => state.global.overridenProperties, - ) const [selectedExtension, setSelectedExtension] = React.useState("") + const selectedExtensionNode = selectedGraph?.nodes.find(n => n.name === selectedExtension) + const addonModules = useAppSelector((state) => state.global.addonModules) + const selectedAddonModule = addonModules.find( + (module) => module.name === selectedExtensionNode?.addon, + ) return ( @@ -246,45 +241,57 @@ export function RemotePropertyCfgSheet() { - {Object.keys(graphExtensions).map((key) => ( - - {key} + {selectedGraph ? (selectedGraph.nodes).map((node) => ( + + {node.name} - ))} + )) : null} - {graphExtensions?.[selectedExtension]?.["property"] && ( + {selectedExtensionNode?.["property"] && ( module.name === selectedExtensionNode?.addon, + )?.api?.property || {} } - onUpdate={(data) => { + onUpdate={async (data) => { // clone the overridenProperties - let nodesMap = JSON.parse( - JSON.stringify(overridenProperties[selectedExtension] || {}), - ) - // Update initial data with any existing overridden values - if (overridenProperties[selectedExtension]) { - Object.assign(nodesMap, overridenProperties[selectedExtension]) + const selectedGraphCopy: Graph = JSON.parse(JSON.stringify(selectedGraph)) + const nodes = selectedGraphCopy?.nodes || [] + let needUpdate = false + for (const node of nodes) { + if (node.name === selectedExtension) { + node.property = data + needUpdate = true + } } - nodesMap[selectedExtension] = data - toast.success("Properties updated", { - description: `Graph: ${graphName}, Extension: ${selectedExtension}`, - }) - dispatch( - setOverridenPropertiesByGraph({ - graphName, - nodesMap, - }), - ) + if (needUpdate) { + await updateGraph(selectedGraphCopy.id, selectedGraphCopy) + toast.success("Properties updated", { + description: `Graph: ${graphName}, Extension: ${selectedExtension}`, + }) + } + + // let nodesMap = JSON.parse( + // JSON.stringify(overridenProperties[selectedExtension] || {}), + // ) + // // Update initial data with any existing overridden values + // if (overridenProperties[selectedExtension]) { + // Object.assign(nodesMap, overridenProperties[selectedExtension]) + // } + // nodesMap[selectedExtension] = data + // toast.success("Properties updated", { + // description: `Graph: ${graphName}, Extension: ${selectedExtension}`, + // }) }} /> )} @@ -299,6 +306,78 @@ export function RemotePropertyCfgSheet() { ) } + + +export function RemotePropertyAddCfgSheet({ + selectedExtension, + extensionNodeData, + onUpdate, +}:{ + selectedExtension: string, + extensionNodeData: Record, + onUpdate: (data: string) => void +}) { + const dispatch = useAppDispatch() + const {selectedGraph} = useGraphManager() + + const selectedExtensionNode = selectedGraph?.nodes.find(n => n.name === selectedExtension) + const addonModules = useAppSelector((state) => state.global.addonModules) + const selectedAddonModule = addonModules.find( + (module) => module.name === selectedExtensionNode?.addon, + ) + const allProperties = Object.keys(selectedAddonModule?.api?.property || {}) + const usedProperties = Object.keys(extensionNodeData) + const remainingProperties = allProperties.filter( + (prop) => !usedProperties.includes(prop), + ) + + const [selectedProperty, setSelectedProperty] = React.useState("") + const [isSheetOpen, setSheetOpen] = React.useState(false) // State to control the sheet + + return ( + + +
+ +
+ + + + Property Add + + You can add a property into a graph extension node and configure its value. + + + + + + + ) +} + // Helper to convert values based on type const convertToType = (value: any, type: string) => { switch (type) { @@ -453,24 +532,31 @@ const GraphModuleCfgForm = ({ ) } +import { useState } from "react" const GraphCfgForm = ({ + selectedExtension, + selectedAddonModule, initialData, metadata, onUpdate, }: { + selectedExtension: string, + selectedAddonModule: AddonDef.Module | undefined, initialData: Record metadata: Record onUpdate: (data: Record) => void }) => { const formSchema = z.record( z.string(), - z.union([z.string(), z.number(), z.boolean(), z.null()]), + z.union([z.string(), z.number(), z.boolean(), z.null()]) ) + const [formData, setFormData] = useState(initialData) + const form = useForm>({ resolver: zodResolver(formSchema), - defaultValues: initialData, + defaultValues: formData, }) const onSubmit = (data: z.infer) => { @@ -480,12 +566,18 @@ const GraphCfgForm = ({ acc[key] = value === "" ? null : convertToType(value, type) return acc }, - {} as Record, + {} as Record ) onUpdate(convertedData) } - const initialDataWithType = Object.entries(initialData).reduce( + const handleDelete = (key: string) => { + const updatedData = { ...formData } + delete updatedData[key] // Remove the specific key + setFormData(updatedData) // Update state + } + + const initialDataWithType = Object.entries(formData).reduce( (acc, [key, value]) => { acc[key] = { value, type: metadata[key]?.type || "string" } return acc @@ -493,7 +585,7 @@ const GraphCfgForm = ({ {} as Record< string, { value: string | number | boolean | null; type: string } - >, + > ) return ( @@ -507,31 +599,58 @@ const GraphCfgForm = ({ render={({ field }) => ( {key} - - {type === "bool" ? ( -
- + + {type === "bool" ? ( +
+ +
+ ) : ( + -
- ) : ( - - )} - + )} + +
handleDelete(key)} // Delete action + > + +
+ )} /> ))} - - - */} ) @@ -312,13 +290,13 @@ export function RemotePropertyAddCfgSheet({ selectedExtension, extensionNodeData, onUpdate, -}:{ +}: { selectedExtension: string, extensionNodeData: Record, onUpdate: (data: string) => void }) { const dispatch = useAppDispatch() - const {selectedGraph} = useGraphManager() + const { selectedGraph } = useGraphManager() const selectedExtensionNode = selectedGraph?.nodes.find(n => n.name === selectedExtension) const addonModules = useAppSelector((state) => state.global.addonModules) @@ -336,10 +314,10 @@ export function RemotePropertyAddCfgSheet({ return ( -
- +
@@ -360,10 +338,10 @@ export function RemotePropertyAddCfgSheet({ {remainingProperties.map((item) => ( - - {item} - - ))} + + {item} + + ))} +
+ { + let defaultProperty = selectedAddonModule?.defaultProperty || {} + let updatedData = { ...formData } + updatedData[key] = defaultProperty[key] + setFormData(updatedData) + }} + /> + +
) From ec0cda0b9357b78a604328db44e4a1b7f8044c7b Mon Sep 17 00:00:00 2001 From: Zhang Qianze Date: Sat, 30 Nov 2024 10:43:46 +0800 Subject: [PATCH 10/28] feat: details --- playground/src/components/Chat/ChatCard.tsx | 2 +- .../src/components/Chat/ChatCfgSelect.tsx | 83 +++++++++++++------ 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/playground/src/components/Chat/ChatCard.tsx b/playground/src/components/Chat/ChatCard.tsx index a729f338..ed4e88e1 100644 --- a/playground/src/components/Chat/ChatCard.tsx +++ b/playground/src/components/Chat/ChatCard.tsx @@ -109,7 +109,7 @@ export default function ChatCard(props: { className?: string }) {
{/* Action Bar */} -
+
diff --git a/playground/src/components/Chat/ChatCfgSelect.tsx b/playground/src/components/Chat/ChatCfgSelect.tsx index 3eed3258..a77fbbd2 100644 --- a/playground/src/components/Chat/ChatCfgSelect.tsx +++ b/playground/src/components/Chat/ChatCfgSelect.tsx @@ -207,6 +207,7 @@ export function RemotePropertyCfgSheet() { const selectedAddonModule = addonModules.find( (module) => module.name === selectedExtensionNode?.addon, ) + const hasProperty = !!selectedAddonModule?.api?.property && Object.keys(selectedAddonModule?.api?.property).length > 0 return ( @@ -246,7 +247,7 @@ export function RemotePropertyCfgSheet() {
- {selectedExtensionNode?.["property"] && ( + {hasProperty ? selectedExtensionNode?.["property"] && ( + ) : ( + + No properties found for the selected extension. + )} @@ -308,6 +313,7 @@ export function RemotePropertyAddCfgSheet({ const remainingProperties = allProperties.filter( (prop) => !usedProperties.includes(prop), ) + const hasRemainingProperties = remainingProperties.length > 0 const [selectedProperty, setSelectedProperty] = React.useState("") const [isSheetOpen, setSheetOpen] = React.useState(false) // State to control the sheet @@ -327,30 +333,45 @@ export function RemotePropertyAddCfgSheet({ You can add a property into a graph extension node and configure its value. - - + {hasRemainingProperties ? ( + <> + + + + + ) : ( + <> + + No remaining properties to add. + + + + )} + ) @@ -621,9 +642,19 @@ const GraphCfgForm = ({ extensionNodeData={formData} onUpdate={(key: string) => { let defaultProperty = selectedAddonModule?.defaultProperty || {} + let defaultValue = defaultProperty[key] + + if (defaultValue === undefined) { + let schema = selectedAddonModule?.api?.property || {} + let schemaType = schema[key]?.type + if (schemaType === "bool") { + defaultValue = false + } + } let updatedData = { ...formData } - updatedData[key] = defaultProperty[key] + updatedData[key] = defaultValue setFormData(updatedData) + form.reset(updatedData) }} />
- - {/* - - - - */} - ) + ); } export function RemotePropertyCfgSheet() { @@ -399,123 +393,62 @@ const GraphModuleCfgForm = ({ metadata, onUpdate, }: { - initialData: Record - metadata: Record - onUpdate: (data: Record) => void + initialData: Record; + metadata: Record; + onUpdate: (data: Record) => void; }) => { - const formSchema = z.object({ - stt: z.string().nullable(), - llm: z.string().nullable(), - tts: z.string().nullable(), - }) + const formSchema = z.record(z.string(), z.string().nullable()); const form = useForm>({ resolver: zodResolver(formSchema), defaultValues: initialData, - }) + }); const onSubmit = (data: z.infer) => { - onUpdate(data) - } + onUpdate(data); + }; + // Custom labels for specific keys + const fieldLabels: Record = { + stt: "STT (Speech to Text)", + llm: "LLM (Large Language Model)", + tts: "TTS (Text to Speech)", + }; return (
- {/* STT Section */} -
-

STT (Speech to Text)

- ( - - Speech-to-Text - - - - - )} - /> -
- - {/* LLM Section */} -
-

LLM (Large Language Model)

- ( - - Large Language Model - - - - - )} - /> -
- - {/* TTS Section */} -
-

TTS (Text to Speech)

- ( - - Text-to-Speech - - - - - )} - /> -
+ {Object.entries(metadata).map(([key, meta]) => ( +
+ ( + + {fieldLabels[key] || key.toUpperCase()} + + + + + )} + /> +
+ ))} - {/* Submit Button */}
- ) -} + ); +}; import { useState } from "react" From 3eb293b1aadffc2d8a80521aca6292da890f0994 Mon Sep 17 00:00:00 2001 From: Zhang Qianze Date: Sat, 30 Nov 2024 12:24:00 +0800 Subject: [PATCH 12/28] feat: request optimization --- playground/src/common/graph.ts | 56 ++++++------ .../src/components/Chat/ChatCfgSelect.tsx | 85 ++++++++++--------- .../src/components/authInitializer/index.tsx | 15 +++- playground/src/store/reducers/global.ts | 45 +++++++++- 4 files changed, 130 insertions(+), 71 deletions(-) diff --git a/playground/src/common/graph.ts b/playground/src/common/graph.ts index 1333567d..4ecda891 100644 --- a/playground/src/common/graph.ts +++ b/playground/src/common/graph.ts @@ -2,9 +2,12 @@ import axios from "axios" import { useCallback, useEffect, useState } from "react" import { useAppDispatch, useAppSelector } from "./hooks" import { + fetchGraphDetails, + initializeGraphData, setAddonModules, setGraph, setGraphList, + updateGraph, } from "@/store/reducers/global" import { apiFetchAddonsExtensions, apiFetchGraphDetails, apiFetchGraphs, apiFetchInstalledAddons, apiReloadPackage, apiSaveProperty, apiUpdateGraph } from "./request" @@ -139,23 +142,19 @@ const useGraphManager = () => { ) const selectedGraph = graphMap[selectedGraphId] - useEffect(() => { + const initialize = async () => { + await dispatch(initializeGraphData()); + }; + + const fetchDetails = async () => { if (selectedGraphId) { - apiFetchGraphDetails(selectedGraphId).then((graph) => { - dispatch(setGraph(graph)) - }) + await dispatch(fetchGraphDetails(selectedGraphId)); } - }, [selectedGraphId]) - - const initializeGraphData = useCallback(async () => { - await apiReloadPackage() - const [fetchedGraphs, modules] = await Promise.all([ - apiFetchGraphs(), - apiFetchInstalledAddons(), - ]) - dispatch(setGraphList(fetchedGraphs.map((graph) => graph.id))) - dispatch(setAddonModules(modules)) - }, []) + }; + + const update = async (graphId: string, updates: Partial) => { + await dispatch(updateGraph({ graphId, updates })); + }; const getGraphNodeAddonByName = useCallback( (nodeName: string) => { @@ -171,23 +170,24 @@ const useGraphManager = () => { [selectedGraph], ) - const updateGraph = useCallback( - async (graphId: string, updates: Partial): Promise => { - await apiUpdateGraph(graphId, updates) + // const updateGraph = useCallback( + // async (graphId: string, updates: Partial): Promise => { + // await apiUpdateGraph(graphId, updates) - // Save additional properties if needed - await apiSaveProperty() + // // Save additional properties if needed + // await apiSaveProperty() - // Update the local state with the latest graph details - const updatedGraph = await apiFetchGraphDetails(graphId) - dispatch(setGraph(updatedGraph)) - }, - [], - ) + // // Update the local state with the latest graph details + // const updatedGraph = await apiFetchGraphDetails(graphId) + // dispatch(setGraph(updatedGraph)) + // }, + // [], + // ) return { - initializeGraphData, - updateGraph, + initialize, + fetchDetails, + update, getGraphNodeAddonByName, selectedGraph, } diff --git a/playground/src/components/Chat/ChatCfgSelect.tsx b/playground/src/components/Chat/ChatCfgSelect.tsx index f69bff05..0496a109 100644 --- a/playground/src/components/Chat/ChatCfgSelect.tsx +++ b/playground/src/components/Chat/ChatCfgSelect.tsx @@ -92,15 +92,17 @@ export function RemoteGraphSelect() { } export function RemoteModuleCfgSheet() { const addonModules = useAppSelector((state) => state.global.addonModules); - const { getGraphNodeAddonByName, selectedGraph, updateGraph } = useGraphManager(); + const { getGraphNodeAddonByName, selectedGraph, update: updateGraph } = useGraphManager(); const modules = React.useMemo(() => { const result: Record = {}; addonModules.forEach((module) => { const matchingNode = selectedGraph?.nodes.find((node) => - ["stt", "tts", "llm"].some((type) => - node.name === type && (module.name.includes(type) || (type === "stt" && module.name.includes("asr"))) + ["stt", "tts", "llm", "llmv2v"].some((type) => + node.name === type && + (module.name.includes(type) || + (type === "stt" && module.name.includes("asr"))) ) ); if (matchingNode) { @@ -148,8 +150,8 @@ export function RemoteModuleCfgSheet() { Module Picker - You can adjust extension modules here, the values will be - written into property.json file. + You can adjust STT/TTS/LLM/LLMv2v extension modules here, the values will be + written into property.json file when you save. @@ -192,7 +194,7 @@ export function RemoteModuleCfgSheet() { export function RemotePropertyCfgSheet() { const dispatch = useAppDispatch() - const { selectedGraph, updateGraph } = useGraphManager() + const { selectedGraph, update: updateGraph } = useGraphManager() const graphName = useAppSelector((state) => state.global.selectedGraphId) const [selectedExtension, setSelectedExtension] = React.useState("") @@ -217,8 +219,8 @@ export function RemotePropertyCfgSheet() { Properties Setting - You can adjust extension properties here, the values will be - written into property.json file. + You can adjust extension properties for selected graph here, the values will be + written into property.json file when you save. @@ -413,41 +415,48 @@ const GraphModuleCfgForm = ({ stt: "STT (Speech to Text)", llm: "LLM (Large Language Model)", tts: "TTS (Text to Speech)", + llmv2v: "LLM v2v (Voice to Voice Large Language Model)", }; + + // Desired field order + const fieldOrder = ["stt", "llm", "llmv2v", "tts"]; return (
- {Object.entries(metadata).map(([key, meta]) => ( -
- ( - - {fieldLabels[key] || key.toUpperCase()} - - - - - )} - /> -
- ))} + {fieldOrder.map( + (key) => + metadata[key] && ( // Check if the field exists in metadata +
+ ( + + {fieldLabels[key]} + + + + + )} + /> +
+ ) + )} +
+ + ); +}; diff --git a/playground/src/components/Chat/ChatCfgSelect.tsx b/playground/src/components/Chat/ChatCfgPropertySelect.tsx similarity index 62% rename from playground/src/components/Chat/ChatCfgSelect.tsx rename to playground/src/components/Chat/ChatCfgPropertySelect.tsx index c7ecc525..43eb202e 100644 --- a/playground/src/components/Chat/ChatCfgSelect.tsx +++ b/playground/src/components/Chat/ChatCfgPropertySelect.tsx @@ -38,14 +38,8 @@ import { Switch } from "@/components/ui/switch" import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { useAppDispatch, - LANGUAGE_OPTIONS, useAppSelector, - GRAPH_OPTIONS, } from "@/common" -import { - setSelectedGraphId, - setLanguage, -} from "@/store/reducers/global" import { cn } from "@/lib/utils" import { SettingsIcon, LoaderCircleIcon, BoxesIcon, Trash2Icon } from "lucide-react" import { zodResolver } from "@hookform/resolvers/zod" @@ -54,164 +48,6 @@ import { z } from "zod" import { toast } from "sonner" import { AddonDef, Graph, useGraphManager } from "@/common/graph" -export function RemoteGraphSelect() { - const dispatch = useAppDispatch() - const graphName = useAppSelector((state) => state.global.selectedGraphId) - const graphs = useAppSelector((state) => state.global.graphList) - const agentConnected = useAppSelector((state) => state.global.agentConnected) - - const onGraphNameChange = (val: string) => { - dispatch(setSelectedGraphId(val)) - } - - const graphOptions = graphs.map((item) => ({ - label: item, - value: item, - })) - - return ( - <> - - - ) -} -export function RemoteModuleCfgSheet() { - const addonModules = useAppSelector((state) => state.global.addonModules); - const { getGraphNodeAddonByName, selectedGraph, update: updateGraph } = useGraphManager(); - - const moduleMapping: Record = { - stt: [], - llm: ["openai_chatgpt_python"], - v2v: [], - tts: [], - }; - - // Define the exclusion map for modules - const exclusionMapping: Record = { - stt: [], - llm: ["qwen_llm_python"], - v2v: ["minimax_v2v_python"], - tts: [], - }; - - const modules = React.useMemo(() => { - const result: Record = {}; - - addonModules.forEach((module) => { - const matchingNode = selectedGraph?.nodes.find((node) => - ["stt", "tts", "llm", "v2v"].some((type) => - node.name === type && - (module.name.includes(type) || - (type === "stt" && module.name.includes("asr")) || - (moduleMapping[type]?.includes(module.name))) - ) - ); - - if ( - matchingNode && - !exclusionMapping[matchingNode.name]?.includes(module.name) - ) { - if (!result[matchingNode.name]) { - result[matchingNode.name] = []; - } - result[matchingNode.name].push(module.name); - } - }); - - return result; - }, [addonModules, selectedGraph]); - - const metadata = React.useMemo(() => { - const dynamicMetadata: Record = {}; - - Object.keys(modules).forEach((key) => { - dynamicMetadata[key] = { type: "string", options: modules[key] }; - }); - - return dynamicMetadata; - }, [modules]); - - const initialData = React.useMemo(() => { - const dynamicInitialData: Record = {}; - - Object.keys(modules).forEach((key) => { - dynamicInitialData[key] = getGraphNodeAddonByName(key)?.addon; - }); - - return dynamicInitialData; - }, [modules, getGraphNodeAddonByName]); - - return ( - - - - - - - Module Picker - - You can adjust STT/TTS/LLM/LLMv2v extension modules here, the values will be - written into property.json file when you save. - - - -
- { - // clone the overriddenAddons - const selectedGraphCopy: Graph = JSON.parse(JSON.stringify(selectedGraph)); - const nodes = selectedGraphCopy?.nodes || []; - let needUpdate = false; - - Object.entries(data).forEach(([key, value]) => { - const node = nodes.find((n) => n.name === key); - if (node && value && node.addon !== value) { - node.addon = value; - node.property = addonModules.find((module) => module.name === value)?.defaultProperty; - needUpdate = true; - } - }); - - if (needUpdate) { - try { - await updateGraph(selectedGraphCopy.id, selectedGraphCopy); - toast.success("Modules updated", { - description: `Graph: ${selectedGraphCopy.id}`, - }); - } catch (e) { - toast.error("Failed to update modules"); - } - } - }} - /> -
- - - ); -} - export function RemotePropertyCfgSheet() { const dispatch = useAppDispatch() const { selectedGraph, update: updateGraph } = useGraphManager() @@ -425,89 +261,6 @@ const defaultTypeValue = (type: string) => { } } -const GraphModuleCfgForm = ({ - initialData, - metadata, - onUpdate, -}: { - initialData: Record; - metadata: Record; - onUpdate: (data: Record) => void; -}) => { - const formSchema = z.record(z.string(), z.string().nullable()); - - const form = useForm>({ - resolver: zodResolver(formSchema), - defaultValues: initialData, - }); - - const onSubmit = (data: z.infer) => { - onUpdate(data); - }; - - // Custom labels for specific keys - const fieldLabels: Record = { - stt: "STT (Speech to Text)", - llm: "LLM (Large Language Model)", - tts: "TTS (Text to Speech)", - v2v: "LLM v2v (Voice to Voice Large Language Model)", - }; - - - // Desired field order - const fieldOrder = ["stt", "llm", "v2v", "tts"]; - return ( -
- - {fieldOrder.map( - (key) => - metadata[key] && ( // Check if the field exists in metadata -
- ( - - {fieldLabels[key]} - - - - - )} - /> -
- ) - )} - - -
- - ); -}; - import { useState } from "react" const GraphCfgForm = ({ From e1b2a053cef4bc1488fee67084e949cf39a6745e Mon Sep 17 00:00:00 2001 From: Zhang Qianze Date: Mon, 2 Dec 2024 05:04:08 +0800 Subject: [PATCH 23/28] feat: support tool binding --- .../bingsearch_tool_python/property.json | 4 +- .../elevenlabs_tts_python/requirements.txt | 3 +- .../openai_chatgpt_python/extension.py | 10 +- .../weatherapi_tool_python/property.json | 4 +- .../interface/ten_ai_base/llm_tool.py | 3 +- .../system/ten_ai_base/requirements.txt | 2 +- playground/package.json | 1 + playground/pnpm-lock.yaml | 82 +++- .../components/Chat/ChatCfgModuleSelect.tsx | 370 ++++++++++++++++-- playground/src/components/ui/dropdown.tsx | 111 ++++++ 10 files changed, 540 insertions(+), 50 deletions(-) create mode 100644 playground/src/components/ui/dropdown.tsx diff --git a/agents/ten_packages/extension/bingsearch_tool_python/property.json b/agents/ten_packages/extension/bingsearch_tool_python/property.json index 9e26dfee..d0cf467d 100644 --- a/agents/ten_packages/extension/bingsearch_tool_python/property.json +++ b/agents/ten_packages/extension/bingsearch_tool_python/property.json @@ -1 +1,3 @@ -{} \ No newline at end of file +{ + "api_key": "${env:BING_API_KEY|}" +} \ No newline at end of file diff --git a/agents/ten_packages/extension/elevenlabs_tts_python/requirements.txt b/agents/ten_packages/extension/elevenlabs_tts_python/requirements.txt index 4346bf71..5e8e39a8 100644 --- a/agents/ten_packages/extension/elevenlabs_tts_python/requirements.txt +++ b/agents/ten_packages/extension/elevenlabs_tts_python/requirements.txt @@ -1,2 +1 @@ -elevenlabs -pydantic<2 \ No newline at end of file +elevenlabs \ No newline at end of file diff --git a/agents/ten_packages/extension/openai_chatgpt_python/extension.py b/agents/ten_packages/extension/openai_chatgpt_python/extension.py index 3b2916d4..81e751f1 100644 --- a/agents/ten_packages/extension/openai_chatgpt_python/extension.py +++ b/agents/ten_packages/extension/openai_chatgpt_python/extension.py @@ -174,10 +174,12 @@ async def on_data_chat_completion(self, ten_env: TenEnv, **kargs: LLMDataComplet self.memory_cache = self.memory_cache + \ [message, {"role": "assistant", "content": ""}] - tools = [] if not no_tool and len( - self.available_tools) > 0 else None - for tool in self.available_tools: - tools.append(self._convert_tools_to_dict(tool)) + tools = None + if not no_tool and len(self.available_tools) > 0: + tools = [] + for tool in self.available_tools: + tools.append(self._convert_tools_to_dict(tool)) + ten_env.log_info(f"tool: {tool}") self.sentence_fragment = "" diff --git a/agents/ten_packages/extension/weatherapi_tool_python/property.json b/agents/ten_packages/extension/weatherapi_tool_python/property.json index 9e26dfee..4f5f409a 100644 --- a/agents/ten_packages/extension/weatherapi_tool_python/property.json +++ b/agents/ten_packages/extension/weatherapi_tool_python/property.json @@ -1 +1,3 @@ -{} \ No newline at end of file +{ + "api_key": "${env:WEATHERAPI_API_KEY|}" +} \ No newline at end of file diff --git a/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm_tool.py b/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm_tool.py index 7d9540ad..4c6f6b07 100644 --- a/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm_tool.py +++ b/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm_tool.py @@ -21,12 +21,13 @@ class AsyncLLMToolBaseExtension(AsyncExtension, ABC): async def on_start(self, ten_env: AsyncTenEnv) -> None: await super().on_start(ten_env) - tools = self.get_tool_metadata(ten_env) + tools:list[LLMToolMetadata] = self.get_tool_metadata(ten_env) for tool in tools: ten_env.log_info(f"tool: {tool}") c: Cmd = Cmd.create(CMD_TOOL_REGISTER) c.set_property_from_json( CMD_PROPERTY_TOOL, json.dumps(tool.model_dump_json())) + ten_env.log_info(f"begin tool register, {tool}") await ten_env.send_cmd(c) ten_env.log_info(f"tool registered, {tool}") diff --git a/agents/ten_packages/system/ten_ai_base/requirements.txt b/agents/ten_packages/system/ten_ai_base/requirements.txt index 9fa23cfd..e9f3515f 100644 --- a/agents/ten_packages/system/ten_ai_base/requirements.txt +++ b/agents/ten_packages/system/ten_ai_base/requirements.txt @@ -1,2 +1,2 @@ -pydantic +pydantic>=2 typing-extensions \ No newline at end of file diff --git a/playground/package.json b/playground/package.json index 84901065..ea8b4569 100644 --- a/playground/package.json +++ b/playground/package.json @@ -17,6 +17,7 @@ "@radix-ui/react-avatar": "^1.1.1", "@radix-ui/react-checkbox": "^1.1.2", "@radix-ui/react-dialog": "^1.1.2", + "@radix-ui/react-dropdown-menu": "^2.1.2", "@radix-ui/react-label": "^2.1.0", "@radix-ui/react-popover": "^1.1.2", "@radix-ui/react-select": "^2.1.2", diff --git a/playground/pnpm-lock.yaml b/playground/pnpm-lock.yaml index bd1d419c..32fc20e8 100644 --- a/playground/pnpm-lock.yaml +++ b/playground/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@radix-ui/react-dialog': specifier: ^1.1.2 version: 1.1.2(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-dropdown-menu': + specifier: ^2.1.2 + version: 2.1.2(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@radix-ui/react-label': specifier: ^2.1.0 version: 2.1.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -868,12 +871,6 @@ packages: cpu: [arm64] os: [linux] - '@img/sharp-linuxmusl-x64@0.33.5': - resolution: {integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - '@img/sharp-wasm32@0.33.5': resolution: {integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} @@ -1231,6 +1228,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-dropdown-menu@2.1.2': + resolution: {integrity: sha512-GVZMR+eqK8/Kes0a36Qrv+i20bAPXSn8rCBTHx30w+3ECnR5o3xixAlqcVaYvLeyKUsm0aqyhWfmUcqufM8nYA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-focus-guards@1.1.1': resolution: {integrity: sha512-pSIwfrT1a6sIoDASCSpFwOasEwKTZWDw/iBdtnqKO7v6FeOzYJ7U53cPzYFVR3geGGXgVHaH+CdngrrAzqUGxg==} peerDependencies: @@ -1275,6 +1285,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-menu@2.1.2': + resolution: {integrity: sha512-lZ0R4qR2Al6fZ4yCCZzu/ReTFrylHFxIqy7OezIpWF4bL0o9biKo0pFIvkaew3TyZ9Fy5gYVrR5zCGZBVbO1zg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-popover@1.1.2': resolution: {integrity: sha512-u2HRUyWW+lOiA2g0Le0tMmT55FGOEWHwPFt1EPfbLly7uXQExFo5duNKqG2DzmFXIdqOeNd+TpE8baHWJCyP9w==} peerDependencies: @@ -4577,11 +4600,6 @@ snapshots: '@img/sharp-libvips-linuxmusl-arm64': 1.0.4 optional: true - '@img/sharp-linuxmusl-x64@0.33.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-x64': 1.0.4 - optional: true - '@img/sharp-wasm32@0.33.5': dependencies: '@emnapi/runtime': 1.3.1 @@ -4882,6 +4900,21 @@ snapshots: '@types/react': 18.3.12 '@types/react-dom': 18.3.1 + '@radix-ui/react-dropdown-menu@2.1.2(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@radix-ui/primitive': 1.1.0 + '@radix-ui/react-compose-refs': 1.1.0(@types/react@18.3.12)(react@18.3.1) + '@radix-ui/react-context': 1.1.1(@types/react@18.3.12)(react@18.3.1) + '@radix-ui/react-id': 1.1.0(@types/react@18.3.12)(react@18.3.1) + '@radix-ui/react-menu': 2.1.2(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-primitive': 2.0.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.1.0(@types/react@18.3.12)(react@18.3.1) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.12 + '@types/react-dom': 18.3.1 + '@radix-ui/react-focus-guards@1.1.1(@types/react@18.3.12)(react@18.3.1)': dependencies: react: 18.3.1 @@ -4915,6 +4948,32 @@ snapshots: '@types/react': 18.3.12 '@types/react-dom': 18.3.1 + '@radix-ui/react-menu@2.1.2(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@radix-ui/primitive': 1.1.0 + '@radix-ui/react-collection': 1.1.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-compose-refs': 1.1.0(@types/react@18.3.12)(react@18.3.1) + '@radix-ui/react-context': 1.1.1(@types/react@18.3.12)(react@18.3.1) + '@radix-ui/react-direction': 1.1.0(@types/react@18.3.12)(react@18.3.1) + '@radix-ui/react-dismissable-layer': 1.1.1(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-focus-guards': 1.1.1(@types/react@18.3.12)(react@18.3.1) + '@radix-ui/react-focus-scope': 1.1.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-id': 1.1.0(@types/react@18.3.12)(react@18.3.1) + '@radix-ui/react-popper': 1.2.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-portal': 1.1.2(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-presence': 1.1.1(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-primitive': 2.0.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-roving-focus': 1.1.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-slot': 1.1.0(@types/react@18.3.12)(react@18.3.1) + '@radix-ui/react-use-callback-ref': 1.1.0(@types/react@18.3.12)(react@18.3.1) + aria-hidden: 1.2.4 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + react-remove-scroll: 2.6.0(@types/react@18.3.12)(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.12 + '@types/react-dom': 18.3.1 + '@radix-ui/react-popover@1.1.2(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: '@radix-ui/primitive': 1.1.0 @@ -7182,7 +7241,6 @@ snapshots: '@img/sharp-linux-s390x': 0.33.5 '@img/sharp-linux-x64': 0.33.5 '@img/sharp-linuxmusl-arm64': 0.33.5 - '@img/sharp-linuxmusl-x64': 0.33.5 '@img/sharp-wasm32': 0.33.5 '@img/sharp-win32-ia32': 0.33.5 '@img/sharp-win32-x64': 0.33.5 diff --git a/playground/src/components/Chat/ChatCfgModuleSelect.tsx b/playground/src/components/Chat/ChatCfgModuleSelect.tsx index 66aae607..64edbd59 100644 --- a/playground/src/components/Chat/ChatCfgModuleSelect.tsx +++ b/playground/src/components/Chat/ChatCfgModuleSelect.tsx @@ -1,42 +1,43 @@ import * as React from "react" import { buttonVariants } from "@/components/ui/button" import { - Select, - SelectContent, - SelectGroup, - SelectItem, - SelectLabel, - SelectTrigger, - SelectValue, + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectLabel, + SelectTrigger, + SelectValue, } from "@/components/ui/select" import { - Sheet, - SheetContent, - SheetDescription, - SheetHeader, - SheetTitle, - SheetTrigger, - SheetFooter, - SheetClose, + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, + SheetTrigger, + SheetFooter, + SheetClose, } from "@/components/ui/sheet" import { - Form, - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage, + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, } from "@/components/ui/form" import { Button } from "@/components/ui/button" import { cn } from "@/lib/utils" import { useAppDispatch, useAppSelector } from "@/common/hooks" import { AddonDef, Graph, useGraphManager } from "@/common/graph" import { toast } from "sonner" -import { BoxesIcon, LoaderCircleIcon } from "lucide-react" +import { BoxesIcon, ChevronRightIcon, LoaderCircleIcon, SettingsIcon, Trash2Icon } from "lucide-react" import { useForm } from "react-hook-form" import { zodResolver } from "@hookform/resolvers/zod" import { z } from "zod" +import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuPortal, DropdownMenuSub, DropdownMenuSubContent, DropdownMenuSubTrigger, DropdownMenuTrigger } from "../ui/dropdown" export function RemoteModuleCfgSheet() { const addonModules = useAppSelector((state) => state.global.addonModules); @@ -127,12 +128,237 @@ export function RemoteModuleCfgSheet() { { - // clone the overriddenAddons + onUpdate={async (data, tools) => { + // Clone the selectedGraph to avoid mutating the original graph const selectedGraphCopy: Graph = JSON.parse(JSON.stringify(selectedGraph)); const nodes = selectedGraphCopy?.nodes || []; + const connections = selectedGraphCopy?.connections || []; let needUpdate = false; + // Retrieve current tools in the graph + const toolModules = addonModules.filter((module) => module.name.includes("tool")); + const currentToolsInGraph = nodes + .filter((node) => toolModules.map((module) => module.name).includes(node.addon)) + .map((node) => node.addon); + + // Retrieve the app value from the agora_rtc node + const agoraRtcNode = nodes.find((node) => node.name === "agora_rtc"); + + if (!agoraRtcNode) { + toast.error("agora_rtc node not found in the graph"); + return; + } + + const agoraApp = agoraRtcNode?.app || "localhost"; + + // Identify removed tools + const removedTools = currentToolsInGraph.filter((tool) => !tools.includes(tool)); + + removedTools.forEach((tool) => { + // Remove the tool node + const toolNodeIndex = nodes.findIndex((node) => node.addon === tool); + if (toolNodeIndex !== -1) { + nodes.splice(toolNodeIndex, 1); + needUpdate = true; + } + + // Remove connections involving the tool + connections.forEach((connection, connIndex) => { + // If the connection extension matches the tool, remove the entire connection + if (connection.extension === tool) { + connections.splice(connIndex, 1); + needUpdate = true; + return; // Skip further processing for this connection + } + + // Remove tool from cmd, data, audioFrame, and videoFrame destinations + const removeEmptyDestObjects = (array: Array<{ name: string; dest: Array }> | undefined) => { + if (!array) return; + + array.forEach((object, objIndex) => { + object.dest = object.dest.filter((dest) => dest.extension !== tool); + + // If `dest` is empty, remove the object + if (object.dest.length === 0) { + array.splice(objIndex, 1); + needUpdate = true; + } + }); + }; + + // Clean up cmd, data, audioFrame, and videoFrame + removeEmptyDestObjects(connection.cmd); + removeEmptyDestObjects(connection.data); + removeEmptyDestObjects(connection.audioFrame); + removeEmptyDestObjects(connection.videoFrame); + + // Remove the entire connection if it has no `cmd`, `data`, `audioFrame`, or `videoFrame` + if ( + (!connection.cmd || connection.cmd.length === 0) && + (!connection.data || connection.data.length === 0) && + (!connection.audioFrame || connection.audioFrame.length === 0) && + (!connection.videoFrame || connection.videoFrame.length === 0) + ) { + connections.splice(connIndex, 1); + needUpdate = true; + } + }); + }); + + // Process tool modules + if (tools.length > 0) { + if (tools.some((tool) => tool.includes("vision"))) { + agoraRtcNode.property = { + ...agoraRtcNode.property, + subscribe_video_pix_fmt: 4, + subscribe_video: true, + } + needUpdate = true; + } else { + delete agoraRtcNode.property?.subscribe_video_pix_fmt; + delete agoraRtcNode.property?.subscribe_video; + } + + tools.forEach((tool) => { + if (!currentToolsInGraph.includes(tool)) { + // 1. Remove existing node for the tool if it exists + const existingNodeIndex = nodes.findIndex((node) => node.name === tool); + if (existingNodeIndex >= 0) { + nodes.splice(existingNodeIndex, 1); + } + + // Add new node for the tool + const toolModule = addonModules.find((module) => module.name === tool); + if (toolModule) { + nodes.push({ + app: agoraApp, + name: tool, + addon: tool, + extensionGroup: "default", + property: toolModule.defaultProperty, + }); + needUpdate = true; + } + + // 2. Find or create a connection for node name "llm" with cmd dest "tool_call" + let llmConnection = connections.find( + (connection) => connection.extension === "llm" + ); + + // Retrieve the extensionGroup dynamically from the graph + const llmNode = nodes.find((node) => node.name === "llm"); + const llmExtensionGroup = llmNode?.extensionGroup || "default"; + + if (llmConnection) { + // If the connection exists, ensure it has a cmd array + if (!llmConnection.cmd) { + llmConnection.cmd = []; + } + + // Find the tool_call command + let toolCallCommand = llmConnection.cmd.find((cmd) => cmd.name === "tool_call"); + + if (!toolCallCommand) { + // If tool_call command doesn't exist, create it + toolCallCommand = { + name: "tool_call", + dest: [], + }; + llmConnection.cmd.push(toolCallCommand); + needUpdate = true; + } + + // Add the tool to the dest array if not already present + if (!toolCallCommand.dest.some((dest) => dest.extension === tool)) { + toolCallCommand.dest.push({ + app: agoraApp, + extensionGroup: "default", + extension: tool, + }); + needUpdate = true; + } + } else { + // If llmConnection doesn't exist, create it with the tool_call command + connections.push({ + app: agoraApp, + extensionGroup: llmExtensionGroup, + extension: "llm", + cmd: [ + { + name: "tool_call", + dest: [ + { + app: agoraApp, + extensionGroup: "default", + extension: tool, + }, + ], + }, + ], + }); + needUpdate = true; + } + + + // 3. Create a connection for the tool node with cmd dest "tool_register" + connections.push({ + app: agoraApp, + extensionGroup: "default", + extension: tool, + cmd: [ + { + name: "tool_register", + dest: [ + { + app: agoraApp, + extensionGroup: llmExtensionGroup, + extension: "llm", + }, + ], + }, + ], + }); + needUpdate = true; + + // Create videoFrame connection for tools with "visual" in the name + if (tool.includes("vision")) { + const rtcConnection = connections.find( + (connection) => + connection.extension === "agora_rtc" + ); + + if (rtcConnection) { + if (!rtcConnection?.videoFrame) { + rtcConnection.videoFrame = [] + } + + if (!rtcConnection.videoFrame.some((frame) => frame.name === "video_frame")) { + rtcConnection.videoFrame.push({ + name: "video_frame", + dest: [ + { + app: agoraApp, + extensionGroup: "default", + extension: tool, + }, + ], + }); + needUpdate = true; + } else if (!rtcConnection.videoFrame.some((frame) => frame.dest.some((dest) => dest.extension === tool))) { + rtcConnection.videoFrame.find((frame) => frame.name === "video_frame")?.dest.push({ + app: agoraApp, + extensionGroup: "default", + extension: tool, + }); + needUpdate = true; + } + } + } + } + }); + } + + // Update graph nodes with selected modules Object.entries(data).forEach(([key, value]) => { const node = nodes.find((n) => n.name === key); if (node && value && node.addon !== value) { @@ -142,6 +368,7 @@ export function RemoteModuleCfgSheet() { } }); + // Perform the update if changes are detected if (needUpdate) { try { await updateGraph(selectedGraphCopy.id, selectedGraphCopy); @@ -153,6 +380,7 @@ export function RemoteModuleCfgSheet() { } } }} + />
@@ -167,9 +395,11 @@ const GraphModuleCfgForm = ({ }: { initialData: Record; metadata: Record; - onUpdate: (data: Record) => void; + onUpdate: (data: Record, tools: string[]) => void; }) => { const formSchema = z.record(z.string(), z.string().nullable()); + const addonModules = useAppSelector((state) => state.global.addonModules); + const { selectedGraph } = useGraphManager(); const form = useForm>({ resolver: zodResolver(formSchema), @@ -177,9 +407,10 @@ const GraphModuleCfgForm = ({ }); const onSubmit = (data: z.infer) => { - onUpdate(data); + onUpdate(data, selectedTools); }; + // Custom labels for specific keys const fieldLabels: Record = { stt: "STT (Speech to Text)", @@ -188,6 +419,21 @@ const GraphModuleCfgForm = ({ v2v: "LLM v2v (Voice to Voice Large Language Model)", }; + // Extract tool modules from addonModules + const toolModules = React.useMemo( + () => addonModules.filter((module) => module.name.includes("tool")), + [addonModules] + ); + + // Initialize selectedTools by extracting tool addons used in graph nodes + const initialSelectedTools = React.useMemo(() => { + const toolNames = toolModules.map((module) => module.name); + return selectedGraph?.nodes + .filter((node) => toolNames.includes(node.addon)) + .map((node) => node.addon) || []; + }, [toolModules, selectedGraph]); + + const [selectedTools, setSelectedTools] = React.useState(initialSelectedTools); // Desired field order const fieldOrder = ["stt", "llm", "v2v", "tts"]; @@ -203,7 +449,50 @@ const GraphModuleCfgForm = ({ name={key} render={({ field }) => ( - {fieldLabels[key]} + +
+
{fieldLabels[key]}
+ {(key === "llm" || key === "v2v") && ( + + + + + + } className="flex justify-between"> + Add Tools + + + + {toolModules.map((module) => ( + { + if (!selectedTools.includes(module.name)) { + setSelectedTools((prev) => [ + ...prev, + module.name, + ]); + } + }}> + {module.name} + + ))} + + + + + + )} +
+