diff --git a/agents/property.json b/agents/property.json index 860a3407..88fab268 100644 --- a/agents/property.json +++ b/agents/property.json @@ -1,4581 +1,4733 @@ { - "_ten": { - "log_level": 3, - "predefined_graphs": [ - { - "name": "va_openai_azure_fashionai", - "auto_start": false, - "connections": [ - { - "data": [ - { - "dest": [ - { - "extension": "interrupt_detector", - "extension_group": "default" - }, - { - "extension": "openai_chatgpt", - "extension_group": "chatgpt" - }, - { - "extension": "message_collector", - "extension_group": "transcriber" - } - ], - "name": "text_data" - } - ], - "extension": "agora_rtc", - "extension_group": "default" - }, - { - "cmd": [ - { - "dest": [ - { - "extension": "azure_tts", - "extension_group": "tts" - }, - { - "extension": "fashionai", - "extension_group": "default" - } - ], - "name": "flush" - } - ], - "data": [ - { - "dest": [ - { - "extension": "message_collector", - "extension_group": "transcriber" - }, - { - "extension": "fashionai", - "extension_group": "default" - } - ], - "name": "text_data" - } - ], - "extension": "openai_chatgpt", - "extension_group": "chatgpt" - }, - { - "audio_frame": [ - { - "dest": [ - { - "extension": "agora_rtc", - "extension_group": "default" - } - ], - "name": "pcm_frame" - } - ], - "cmd": [ - { - "dest": [ - { - "extension": "agora_rtc", - "extension_group": "default" - } - ], - "name": "flush" - } - ], - "extension": "azure_tts", - "extension_group": "tts" - }, - { - "data": [ - { - "dest": [ - { - "extension": "agora_rtc", - "extension_group": "default" - } - ], - "name": "data" - } - ], - "extension": "message_collector", - "extension_group": "transcriber" - }, - { - "cmd": [ - { - "dest": [ - { - "extension": "openai_chatgpt", - "extension_group": "chatgpt" - } + "_ten": { + "log_level": 3, + "predefined_graphs": [ + { + "name": "va_openai_azure_fashionai", + "auto_start": false, + "connections": [ + { + "data": [ + { + "dest": [ + { + "extension": "interrupt_detector", + "extension_group": "default" + }, + { + "extension": "openai_chatgpt", + "extension_group": "chatgpt" + }, + { + "extension": "message_collector", + "extension_group": "transcriber" + } + ], + "name": "text_data" + } + ], + "extension": "agora_rtc", + "extension_group": "default" + }, + { + "cmd": [ + { + "dest": [ + { + "extension": "azure_tts", + "extension_group": "tts" + }, + { + "extension": "fashionai", + "extension_group": "default" + } + ], + "name": "flush" + } + ], + "data": [ + { + "dest": [ + { + "extension": "message_collector", + "extension_group": "transcriber" + }, + { + "extension": "fashionai", + "extension_group": "default" + } + ], + "name": "text_data" + } + ], + "extension": "openai_chatgpt", + "extension_group": "chatgpt" + }, + { + "audio_frame": [ + { + "dest": [ + { + "extension": "agora_rtc", + "extension_group": "default" + } + ], + "name": "pcm_frame" + } + ], + "cmd": [ + { + "dest": [ + { + "extension": "agora_rtc", + "extension_group": "default" + } + ], + "name": "flush" + } + ], + "extension": "azure_tts", + "extension_group": "tts" + }, + { + "data": [ + { + "dest": [ + { + "extension": "agora_rtc", + "extension_group": "default" + } + ], + "name": "data" + } + ], + "extension": "message_collector", + "extension_group": "transcriber" + }, + { + "cmd": [ + { + "dest": [ + { + "extension": "openai_chatgpt", + "extension_group": "chatgpt" + } + ], + "name": "flush" + } + ], + "extension": "interrupt_detector", + "extension_group": "default" + } ], - "name": "flush" - } - ], - "extension": "interrupt_detector", - "extension_group": "default" - } - ], - "nodes": [ - { - "addon": "agora_rtc", - "extension_group": "default", - "name": "agora_rtc", - "property": { - "agora_asr_language": "en-US", - "agora_asr_session_control_file_path": "session_control.conf", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_name": "microsoft", - "agora_asr_vendor_region": "$AZURE_STT_REGION", - "app_id": "$AGORA_APP_ID", - "channel": "astra_agents_test", - "enable_agora_asr": true, - "publish_audio": true, - "publish_data": true, - "remote_stream_id": 123, - "stream_id": 1234, - "subscribe_audio": true, - "token": "" - }, - "type": "extension" - }, - { - "addon": "interrupt_detector", - "extension_group": "default", - "name": "interrupt_detector", - "type": "extension" - }, - { - "addon": "openai_chatgpt", - "extension_group": "chatgpt", - "name": "openai_chatgpt", - "property": { - "api_key": "${env:OPENAI_API_KEY}", - "base_url": "${env:OPENAI_API_BASE}", - "frequency_penalty": 0.9, - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10, - "max_tokens": 512, - "model": "${env:OPENAI_MODEL}", - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}" - }, - "type": "extension" - }, - { - "addon": "azure_tts", - "extension_group": "tts", - "name": "azure_tts", - "property": { - "azure_subscription_key": "$AZURE_TTS_KEY", - "azure_subscription_region": "$AZURE_TTS_REGION", - "azure_synthesis_voice_name": "en-US-JaneNeural" - }, - "type": "extension" - }, - { - "addon": "message_collector", - "extension_group": "transcriber", - "name": "message_collector", - "type": "extension" - }, - { - "addon": "fashionai", - "extension_group": "default", - "name": "fashionai", - "property": { - "app_id": "$AGORA_APP_ID", - "channel": "astra_agents_test", - "stream_id": 12345, - "token": "", - "service_id": "agora" - }, - "type": "extension" - } - ] - }, - { - "name": "va_openai_azure", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt", - "name": "openai_chatgpt", - "property": { - "base_url": "${env:OPENAI_API_BASE}", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "${env:OPENAI_MODEL}", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "azure_tts", - "name": "azure_tts", - "property": { - "azure_subscription_key": "${env:AZURE_TTS_KEY}", - "azure_subscription_region": "${env:AZURE_TTS_REGION}", - "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "azure_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - } - ] - }, - { - "name": "va_simple_azure_openai_v2v", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000 - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:AZURE_OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "voice": "alloy", - "language": "en-US", - "server_vad": true, - "dump": true, - "history": 10, - "vendor": "azure", - "base_uri": "${env:AZURE_OPENAI_REALTIME_BASE_URI}", - "path": "/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview", - "system_message": "" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "openai_v2v_python", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "va_openai_11labs", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt", - "name": "openai_chatgpt", - "property": { - "base_url": "${env:OPENAI_API_BASE}", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "${env:OPENAI_MODEL}", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "elevenlabs_tts", - "name": "elevenlabs_tts", - "property": { - "api_key": "${env:ELEVENLABS_TTS_KEY}", - "model_id": "eleven_multilingual_v2", - "optimize_streaming_latency": 0, - "request_timeout_seconds": 30, - "similarity_boost": 0.75, - "speaker_boost": false, - "stability": 0.5, - "style": 0.0, - "voice_id": "pNInz6obpgDQGcFmaJgB" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "elevenlabs_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "elevenlabs_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "elevenlabs_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - } - ] - }, - { - "name": "va_openai_fish", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "astra_agents_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt", - "name": "openai_chatgpt", - "property": { - "base_url": "${env:OPENAI_API_BASE}", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "${env:OPENAI_MODEL}", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "fish_audio_tts", - "name": "fish_audio_tts", - "property": { - "api_key": "${env:FISH_AUDIO_TTS_KEY}", - "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", - "optimize_streaming_latency": true, - "request_timeout_seconds": 30, - "base_url": "https://api.fish.audio" - } - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "fish_audio_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "fish_audio_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "fish_audio_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - } - ] - }, - { - "name": "va_bedrock_azure", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "bedrock", - "addon": "bedrock_llm_python", - "name": "bedrock_llm", - "property": { - "region": "us-east-1", - "access_key": "${env:AWS_ACCESS_KEY_ID}", - "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", - "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", - "max_tokens": 512, - "prompt": "", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "azure_tts", - "name": "azure_tts", - "property": { - "azure_subscription_key": "${env:AZURE_TTS_KEY}", - "azure_subscription_region": "${env:AZURE_TTS_REGION}", - "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector_python", - "name": "interrupt_detector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "bedrock", - "extension": "bedrock_llm" - } - ] - } - ] - }, - { - "extension_group": "bedrock", - "extension": "bedrock_llm", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "azure_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "bedrock", - "extension": "bedrock_llm" - } - ] - } - ] - } - ] - }, - { - "name": "va_openai_cosy", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt_python", - "name": "openai_chatgpt", - "property": { - "base_url": "${env:OPENAI_API_BASE}", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "${env:OPENAI_MODEL}", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "cosy_tts", - "name": "cosy_tts", - "property": { - "api_key": "${env:QWEN_API_KEY}", - "model": "cosyvoice-v1", - "voice": "longxiaochun", - "sample_rate": 16000 - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector_python", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "chat_transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - }, - { - "extension_group": "chat_transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "on_user_joined", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - }, - { - "name": "on_user_left", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "cosy_tts" - }, - { - "extension_group": "chat_transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "cosy_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "cosy_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - }, - { - "extension_group": "chat_transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "va_qwen_cosy", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "qwen_llm_python", - "name": "qwen_llm", - "property": { - "api_key": "${env:QWEN_API_KEY}", - "model": "qwen-max", - "max_tokens": 512, - "prompt": "", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "cosy_tts", - "name": "cosy_tts", - "property": { - "api_key": "${env:QWEN_API_KEY}", - "model": "cosyvoice-v1", - "voice": "longxiaochun", - "sample_rate": 16000 - } - }, - { - "type": "extension", - "extension_group": "chat_transcriber", - "addon": "message_collector", - "name": "message_collector" - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector_python", - "name": "interrupt_detector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chat_transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "llm", - "extension": "qwen_llm" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "llm", - "extension": "qwen_llm" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "qwen_llm", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "cosy_tts" - }, - { - "extension_group": "chat_transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "cosy_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "cosy_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "chat_transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "va_bedrock_polly", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "bedrock", - "addon": "bedrock_llm_python", - "name": "bedrock_llm", - "property": { - "region": "us-east-1", - "access_key": "${env:AWS_ACCESS_KEY_ID}", - "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", - "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", - "max_tokens": 512, - "prompt": "", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "polly_tts", - "name": "polly_tts", - "property": { - "region": "us-east-1", - "access_key": "${env:AWS_ACCESS_KEY_ID}", - "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", - "engine": "generative", - "voice": "Ruth", - "sample_rate": "16000", - "lang_code": "en-US" - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector_python", - "name": "interrupt_detector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "bedrock", - "extension": "bedrock_llm" - } - ] - } - ] - }, - { - "extension_group": "bedrock", - "extension": "bedrock_llm", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "polly_tts" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "polly_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "polly_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "bedrock", - "extension": "bedrock_llm" - } - ] - } - ] - } - ] - }, - { - "name": "va_transcribe_bedrock_polly", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": false, - "enable_agora_asr": false, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "asr", - "addon": "transcribe_asr_python", - "name": "transcribe_asr", - "property": { - "region": "us-east-1", - "access_key": "${env:AWS_ACCESS_KEY_ID}", - "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", - "sample_rate": "16000", - "lang_code": "en-US" - } - }, - { - "type": "extension", - "extension_group": "bedrock", - "addon": "bedrock_llm_python", - "name": "bedrock_llm", - "property": { - "region": "us-east-1", - "access_key": "${env:AWS_ACCESS_KEY_ID}", - "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", - "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", - "max_tokens": 512, - "prompt": "", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "polly_tts", - "name": "polly_tts", - "property": { - "region": "us-east-1", - "access_key": "${env:AWS_ACCESS_KEY_ID}", - "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", - "engine": "generative", - "voice": "Ruth", - "sample_rate": "16000", - "lang_code": "en-US" - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector_python", - "name": "interrupt_detector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "asr", - "extension": "transcribe_asr" - } - ] - } - ], - "cmd": [ - { - "name": "on_user_joined", - "dest": [ - { - "extension_group": "asr", - "extension": "transcribe_asr" - } - ] - }, - { - "name": "on_user_left", - "dest": [ - { - "extension_group": "asr", - "extension": "transcribe_asr" - } - ] - }, - { - "name": "on_connection_failure", - "dest": [ - { - "extension_group": "asr", - "extension": "transcribe_asr" - } - ] - } - ] - }, - { - "extension_group": "asr", - "extension": "transcribe_asr", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "bedrock", - "extension": "bedrock_llm" - } - ] - } - ] - }, - { - "extension_group": "bedrock", - "extension": "bedrock_llm", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "polly_tts" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "polly_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "polly_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "bedrock", - "extension": "bedrock_llm" - } - ] - } - ] - } - ] - }, - { - "name": "camera_va_openai_azure", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "subscribe_video": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf", - "subscribe_video_pix_fmt": 4 - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt_python", - "name": "openai_chatgpt", - "property": { - "base_url": "${env:OPENAI_API_BASE}", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "${env:OPENAI_MODEL}", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", - "max_memory_length": 10, - "enable_tools": true - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "azure_tts", - "name": "azure_tts", - "property": { - "azure_subscription_key": "${env:AZURE_TTS_KEY}", - "azure_subscription_region": "${env:AZURE_TTS_REGION}", - "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "video_frame": [ - { - "name": "video_frame", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ], - "cmd": [ - { - "name": "on_user_joined", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - }, - { - "name": "on_user_left", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "azure_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - } - ] - }, - { - "name": "va_gemini_azure", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "gemini_llm_python", - "name": "gemini_llm", - "property": { - "api_key": "$GEMINI_API_KEY", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10, - "max_output_tokens": 512, - "model": "gemini-1.5-flash", - "prompt": "", - "temperature": 1.0, - "top_k": 40, - "top_p": 0.95 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "azure_tts", - "name": "azure_tts", - "property": { - "azure_subscription_key": "${env:AZURE_TTS_KEY}", - "azure_subscription_region": "${env:AZURE_TTS_REGION}", - "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "llm", - "extension": "gemini_llm" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "gemini_llm", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "azure_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "llm", - "extension": "gemini_llm" - } - ] - } - ] - } - ] - }, - { - "name": "va_qwen_rag", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "qwen_llm_python", - "name": "qwen_llm", - "property": { - "api_key": "${env:QWEN_API_KEY}", - "model": "qwen-max", - "max_tokens": 512, - "prompt": "", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "cosy_tts", - "name": "cosy_tts", - "property": { - "api_key": "${env:QWEN_API_KEY}", - "model": "cosyvoice-v1", - "voice": "longxiaochun", - "sample_rate": 16000 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "azure_tts", - "name": "azure_tts", - "property": { - "azure_subscription_key": "${env:AZURE_TTS_KEY}", - "azure_subscription_region": "${env:AZURE_TTS_REGION}", - "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" - } - }, - { - "type": "extension", - "extension_group": "chat_transcriber", - "addon": "message_collector", - "name": "message_collector" - }, - { - "type": "extension", - "extension_group": "interrupt_detector", - "addon": "interrupt_detector_python", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "http_server", - "addon": "http_server_python", - "name": "http_server", - "property": { - "listen_addr": "127.0.0.1", - "listen_port": 8080 - } - }, - { - "type": "extension", - "extension_group": "embedding", - "addon": "aliyun_text_embedding", - "name": "aliyun_text_embedding", - "property": { - "api_key": "${env:ALIYUN_TEXT_EMBEDDING_API_KEY}", - "model": "text-embedding-v3" - } - }, - { - "type": "extension", - "extension_group": "vector_storage", - "addon": "aliyun_analyticdb_vector_storage", - "name": "aliyun_analyticdb_vector_storage", - "property": { - "alibaba_cloud_access_key_id": "${env:ALIBABA_CLOUD_ACCESS_KEY_ID}", - "alibaba_cloud_access_key_secret": "${env:ALIBABA_CLOUD_ACCESS_KEY_SECRET}", - "adbpg_instance_id": "${env:ALIYUN_ANALYTICDB_INSTANCE_ID}", - "adbpg_instance_region": "${env:ALIYUN_ANALYTICDB_INSTANCE_REGION}", - "adbpg_account": "${env:ALIYUN_ANALYTICDB_ACCOUNT}", - "adbpg_account_password": "${env:ALIYUN_ANALYTICDB_ACCOUNT_PASSWORD}", - "adbpg_namespace": "${env:ALIYUN_ANALYTICDB_NAMESPACE}", - "adbpg_namespace_password": "${env:ALIYUN_ANALYTICDB_NAMESPACE_PASSWORD}" - } - }, - { - "type": "extension", - "extension_group": "file_chunker", - "addon": "file_chunker", - "name": "file_chunker", - "property": {} - }, - { - "type": "extension", - "extension_group": "llama_index", - "addon": "llama_index_chat_engine", - "name": "llama_index", - "property": { - "greeting": "TEN Agent connected. How can I help you today?", - "chat_memory_token_limit": 3000 - } - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "interrupt_detector", - "extension": "interrupt_detector" - }, - { - "extension_group": "chat_transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "interrupt_detector", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "llama_index", - "extension": "llama_index" - } - ] - }, - { - "name": "file_chunk", - "dest": [ - { - "extension_group": "file_chunker", - "extension": "file_chunker" - }, - { - "extension_group": "llama_index", - "extension": "llama_index" - } - ] - }, - { - "name": "file_chunked", - "dest": [ - { - "extension_group": "llama_index", - "extension": "llama_index" - } - ] - }, - { - "name": "update_querying_collection", - "dest": [ - { - "extension_group": "llama_index", - "extension": "llama_index" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "llama_index", - "extension": "llama_index" - } - ] - } - ] - }, - { - "extension_group": "llama_index", - "extension": "llama_index", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - }, - { - "extension_group": "chat_transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "llm", - "extension": "qwen_llm" - }, - { - "extension_group": "tts", - "extension": "azure_tts" - } - ] - }, - { - "name": "call_chat", - "dest": [ - { - "extension_group": "llm", - "extension": "qwen_llm" - } - ] - }, - { - "name": "embed", - "dest": [ - { - "extension_group": "embedding", - "extension": "aliyun_text_embedding" - } - ] - }, - { - "name": "query_vector", - "dest": [ - { - "extension_group": "vector_storage", - "extension": "aliyun_analyticdb_vector_storage" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "azure_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "chat_transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "http_server", - "extension": "http_server", - "cmd": [ - { - "name": "file_chunk", - "dest": [ - { - "extension_group": "interrupt_detector", - "extension": "interrupt_detector" - } - ] - }, - { - "name": "update_querying_collection", - "dest": [ - { - "extension_group": "interrupt_detector", - "extension": "interrupt_detector" - } - ] - } - ] - }, - { - "extension_group": "file_chunker", - "extension": "file_chunker", - "cmd": [ - { - "name": "embed_batch", - "dest": [ - { - "extension_group": "embedding", - "extension": "aliyun_text_embedding" - } - ] - }, - { - "name": "create_collection", - "dest": [ - { - "extension_group": "vector_storage", - "extension": "aliyun_analyticdb_vector_storage" - } - ] - }, - { - "name": "upsert_vector", - "dest": [ - { - "extension_group": "vector_storage", - "extension": "aliyun_analyticdb_vector_storage" - } - ] - }, - { - "name": "file_chunked", - "dest": [ - { - "extension_group": "llama_index", - "extension": "llama_index" - } - ] - } - ] - } - ] - }, - { - "name": "va_simple_openai_v2v", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000 - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "voice": "alloy", - "language": "en-US", - "server_vad": true, - "dump": true, - "history": 10 - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "openai_v2v_python", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "va_openai_v2v", - "auto_start": true, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000 - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "voice": "alloy", - "language": "en-US", - "server_vad": true, - "dump": true, - "history": 10 - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - }, - { - "type": "extension", - "extension_group": "tools", - "addon": "weatherapi_tool_python", - "name": "weatherapi_tool_python", - "property": { - "api_key": "${env:WEATHERAPI_API_KEY}" - } - }, - { - "type": "extension", - "extension_group": "tools", - "addon": "bingsearch_tool_python", - "name": "bingsearch_tool_python", - "property": { - "api_key": "${env:BING_API_KEY}" - } - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "cmd": [ - { - "name": "tool_register", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "tools", - "extension": "bingsearch_tool_python", - "cmd": [ - { - "name": "tool_register", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "openai_v2v_python", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } + "nodes": [ + { + "addon": "agora_rtc", + "extension_group": "default", + "name": "agora_rtc", + "property": { + "agora_asr_language": "en-US", + "agora_asr_session_control_file_path": "session_control.conf", + "agora_asr_vendor_key": "$AZURE_STT_KEY", + "agora_asr_vendor_name": "microsoft", + "agora_asr_vendor_region": "$AZURE_STT_REGION", + "app_id": "$AGORA_APP_ID", + "channel": "astra_agents_test", + "enable_agora_asr": true, + "publish_audio": true, + "publish_data": true, + "remote_stream_id": 123, + "stream_id": 1234, + "subscribe_audio": true, + "token": "" + }, + "type": "extension" + }, + { + "addon": "interrupt_detector", + "extension_group": "default", + "name": "interrupt_detector", + "type": "extension" + }, + { + "addon": "openai_chatgpt", + "extension_group": "chatgpt", + "name": "openai_chatgpt", + "property": { + "api_key": "${env:OPENAI_API_KEY}", + "base_url": "${env:OPENAI_API_BASE}", + "frequency_penalty": 0.9, + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10, + "max_tokens": 512, + "model": "${env:OPENAI_MODEL}", + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}" + }, + "type": "extension" + }, + { + "addon": "azure_tts", + "extension_group": "tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "$AZURE_TTS_KEY", + "azure_subscription_region": "$AZURE_TTS_REGION", + "azure_synthesis_voice_name": "en-US-JaneNeural" + }, + "type": "extension" + }, + { + "addon": "message_collector", + "extension_group": "transcriber", + "name": "message_collector", + "type": "extension" + }, + { + "addon": "fashionai", + "extension_group": "default", + "name": "fashionai", + "property": { + "app_id": "$AGORA_APP_ID", + "channel": "astra_agents_test", + "stream_id": 12345, + "token": "", + "service_id": "agora" + }, + "type": "extension" + } ] - }, - { - "name": "tool_call_get_current_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_openai_azure", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt", + "name": "openai_chatgpt", + "property": { + "base_url": "${env:OPENAI_API_BASE}", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 } - ] + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", + "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] } - } ] - }, - { - "name": "tool_call_get_past_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_simple_azure_openai_v2v", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "subscribe_audio_sample_rate": 24000 + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "openai_v2v_python", + "name": "openai_v2v_python", + "property": { + "api_key": "${env:AZURE_OPENAI_REALTIME_API_KEY}", + "temperature": 0.9, + "model": "gpt-4o-realtime-preview", + "max_tokens": 2048, + "voice": "alloy", + "language": "en-US", + "server_vad": true, + "dump": true, + "history": 10, + "vendor": "azure", + "base_uri": "${env:AZURE_OPENAI_REALTIME_BASE_URI}", + "path": "/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview", + "system_message": "" } - ] + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "openai_v2v_python", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] } - } ] - }, - { - "name": "tool_call_get_future_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_openai_11labs", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt", + "name": "openai_chatgpt", + "property": { + "base_url": "${env:OPENAI_API_BASE}", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 } - ] + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "elevenlabs_tts", + "name": "elevenlabs_tts", + "property": { + "api_key": "${env:ELEVENLABS_TTS_KEY}", + "model_id": "eleven_multilingual_v2", + "optimize_streaming_latency": 0, + "request_timeout_seconds": 30, + "similarity_boost": 0.75, + "speaker_boost": false, + "stability": 0.5, + "style": 0.0, + "voice_id": "pNInz6obpgDQGcFmaJgB" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "elevenlabs_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "elevenlabs_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "elevenlabs_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] } - } ] - }, - { - "name": "tool_call_bing_search", - "dest": [ - { - "extension_group": "tools", - "extension": "bingsearch_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_openai_fish", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt", + "name": "openai_chatgpt", + "property": { + "base_url": "${env:OPENAI_API_BASE}", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "fish_audio_tts", + "name": "fish_audio_tts", + "property": { + "api_key": "${env:FISH_AUDIO_TTS_KEY}", + "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", + "optimize_streaming_latency": true, + "request_timeout_seconds": 30, + "base_url": "https://api.fish.audio" } - ] } - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "va_simple_openai_v2v_fish", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "audio_out": false, - "input_transcript": false, - "language": "en-US", - "server_vad": true, - "dump": true, - "history": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "fish_audio_tts", - "name": "fish_audio_tts", - "property": { - "api_key": "${env:FISH_AUDIO_TTS_KEY}", - "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", - "optimize_streaming_latency": true, - "request_timeout_seconds": 30, - "base_url": "https://api.fish.audio" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "openai_v2v_python", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - }, - { - "extension_group": "tts", - "extension": "fish_audio_tts" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "fish_audio_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "fish_audio_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "va_openai_v2v_fish", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "audio_out": false, - "input_transcript": false, - "language": "en-US", - "server_vad": true, - "dump": true, - "history": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "fish_audio_tts", - "name": "fish_audio_tts", - "property": { - "api_key": "${env:FISH_AUDIO_TTS_KEY}", - "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", - "optimize_streaming_latency": true, - "request_timeout_seconds": 30, - "base_url": "https://api.fish.audio" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - }, - { - "type": "extension", - "extension_group": "tools", - "addon": "weatherapi_tool_python", - "name": "weatherapi_tool_python", - "property": { - "api_key": "${env:WEATHERAPI_API_KEY}" - } - }, - { - "type": "extension", - "extension_group": "tools", - "addon": "bingsearch_tool_python", - "name": "bingsearch_tool_python", - "property": { - "api_key": "${env:BING_API_KEY}" - } - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ], - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "cmd": [ - { - "name": "tool_register", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "tools", - "extension": "bingsearch_tool_python", - "cmd": [ - { - "name": "tool_register", - "dest": [ - { - "extension_group": "llm", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "openai_v2v_python", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - }, - { - "extension_group": "tts", - "extension": "fish_audio_tts" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "fish_audio_tts" - } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "fish_audio_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "fish_audio_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "fish_audio_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + } ] - }, - { - "name": "tool_call_get_current_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_bedrock_azure", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "bedrock", + "addon": "bedrock_llm_python", + "name": "bedrock_llm", + "property": { + "region": "us-east-1", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", + "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "max_tokens": 512, + "prompt": "", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 } - ] + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", + "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] } - } ] - }, - { - "name": "tool_call_get_past_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_openai_cosy", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt_python", + "name": "openai_chatgpt", + "property": { + "base_url": "${env:OPENAI_API_BASE}", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "cosy_tts", + "name": "cosy_tts", + "property": { + "api_key": "${env:QWEN_API_KEY}", + "model": "cosyvoice-v1", + "voice": "longxiaochun", + "sample_rate": 16000 } - ] + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chat_transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "chat_transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "cosy_tts" + }, + { + "extension_group": "chat_transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "cosy_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "cosy_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + }, + { + "extension_group": "chat_transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] } - } ] - }, - { - "name": "tool_call_get_future_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_qwen_cosy", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "qwen_llm_python", + "name": "qwen_llm", + "property": { + "api_key": "${env:QWEN_API_KEY}", + "model": "qwen-max", + "max_tokens": 512, + "prompt": "", + "max_memory_length": 10 } - ] + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "cosy_tts", + "name": "cosy_tts", + "property": { + "api_key": "${env:QWEN_API_KEY}", + "model": "cosyvoice-v1", + "voice": "longxiaochun", + "sample_rate": 16000 + } + }, + { + "type": "extension", + "extension_group": "chat_transcriber", + "addon": "message_collector", + "name": "message_collector" + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chat_transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "qwen_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "cosy_tts" + }, + { + "extension_group": "chat_transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "cosy_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "cosy_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "chat_transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] } - } ] - }, - { - "name": "tool_call_bing_search", - "dest": [ - { - "extension_group": "tools", - "extension": "bingsearch_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_bedrock_polly", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "bedrock", + "addon": "bedrock_llm_python", + "name": "bedrock_llm", + "property": { + "region": "us-east-1", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", + "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "max_tokens": 512, + "prompt": "", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "polly_tts", + "name": "polly_tts", + "property": { + "region": "us-east-1", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", + "engine": "generative", + "voice": "Ruth", + "sample_rate": "16000", + "lang_code": "en-US" } - ] + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "polly_tts" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "polly_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "polly_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] } - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "fish_audio_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "va_openai_v2v_storage", - "auto_start": true, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000 - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "voice": "alloy", - "language": "en-US", - "server_vad": true, - "dump": true, - "history": 10, - "enable_storage": true - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - }, - { - "type": "extension", - "extension_group": "tools", - "addon": "weatherapi_tool_python", - "name": "weatherapi_tool_python", - "property": { - "api_key": "${env:WEATHERAPI_API_KEY}" - } - }, - { - "type": "extension", - "extension_group": "tools", - "addon": "bingsearch_tool_python", - "name": "bingsearch_tool_python", - "property": { - "api_key": "${env:BING_API_KEY}" - } - }, - { - "type": "extension", - "extension_group": "context", - "addon": "tsdb_firestore", - "name": "tsdb_firestore", - "property": { - "credentials": { - "type": "service_account", - "project_id": "${env:FIRESTORE_PROJECT_ID}", - "private_key_id": "${env:FIRESTORE_PRIVATE_KEY_ID}", - "private_key": "${env:FIRESTORE_PRIVATE_KEY}", - "client_email": "${env:FIRESTORE_CLIENT_EMAIL}", - "client_id": "${env:FIRESTORE_CLIENT_ID}", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "${env:FIRESTORE_CERT_URL}", - "universe_domain": "googleapis.com" - }, - "channel_name": "ten_agent_test", - "collection_name": "llm_context" - } - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "realtime", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "cmd": [ - { - "name": "tool_register", - "dest": [ - { - "extension_group": "realtime", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "tools", - "extension": "bingsearch_tool_python", - "cmd": [ - { - "name": "tool_register", - "dest": [ - { - "extension_group": "realtime", - "extension": "openai_v2v_python" - } - ] - } - ] - }, - { - "extension_group": "realtime", - "extension": "openai_v2v_python", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "data": [ - { - "name": "append", - "dest": [ - { - "extension_group": "context", - "extension": "tsdb_firestore" - } - ] - }, - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - }, - { - "name": "retrieve", - "dest": [ - { - "extension_group": "context", - "extension": "tsdb_firestore" - } ] - }, - { - "name": "tool_call_get_current_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_transcribe_bedrock_polly", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": false, + "enable_agora_asr": false, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "asr", + "addon": "transcribe_asr_python", + "name": "transcribe_asr", + "property": { + "region": "us-east-1", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", + "sample_rate": "16000", + "lang_code": "en-US" + } + }, + { + "type": "extension", + "extension_group": "bedrock", + "addon": "bedrock_llm_python", + "name": "bedrock_llm", + "property": { + "region": "us-east-1", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", + "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "max_tokens": 512, + "prompt": "", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "polly_tts", + "name": "polly_tts", + "property": { + "region": "us-east-1", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", + "engine": "generative", + "voice": "Ruth", + "sample_rate": "16000", + "lang_code": "en-US" } - ] + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "asr", + "extension": "transcribe_asr" + } + ] + } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "asr", + "extension": "transcribe_asr" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "asr", + "extension": "transcribe_asr" + } + ] + }, + { + "name": "on_connection_failure", + "dest": [ + { + "extension_group": "asr", + "extension": "transcribe_asr" + } + ] + } + ] + }, + { + "extension_group": "asr", + "extension": "transcribe_asr", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "polly_tts" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "polly_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "polly_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] } - } ] - }, - { - "name": "tool_call_get_past_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "camera_va_openai_azure", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "subscribe_video": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf", + "subscribe_video_pix_fmt": 4 + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt_python", + "name": "openai_chatgpt", + "property": { + "base_url": "${env:OPENAI_API_BASE}", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}", + "greeting": "TEN Agent connected. How can I help you today?", + "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", + "max_memory_length": 10, + "enable_tools": true } - ] + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", + "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "video_frame": [ + { + "name": "video_frame", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] } - } ] - }, - { - "name": "tool_call_get_future_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_gemini_azure", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "gemini_llm_python", + "name": "gemini_llm", + "property": { + "api_key": "$GEMINI_API_KEY", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10, + "max_output_tokens": 512, + "model": "gemini-1.5-flash", + "prompt": "", + "temperature": 1.0, + "top_k": 40, + "top_p": 0.95 } - ] + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", + "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "llm", + "extension": "gemini_llm" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "gemini_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llm", + "extension": "gemini_llm" + } + ] + } + ] } - } ] - }, - { - "name": "tool_call_bing_search", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" + }, + { + "name": "va_qwen_rag", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "qwen_llm_python", + "name": "qwen_llm", + "property": { + "api_key": "${env:QWEN_API_KEY}", + "model": "qwen-max", + "max_tokens": 512, + "prompt": "", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "cosy_tts", + "name": "cosy_tts", + "property": { + "api_key": "${env:QWEN_API_KEY}", + "model": "cosyvoice-v1", + "voice": "longxiaochun", + "sample_rate": 16000 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", + "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + } + }, + { + "type": "extension", + "extension_group": "chat_transcriber", + "addon": "message_collector", + "name": "message_collector" + }, + { + "type": "extension", + "extension_group": "interrupt_detector", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "http_server", + "addon": "http_server_python", + "name": "http_server", + "property": { + "listen_addr": "127.0.0.1", + "listen_port": 8080 + } + }, + { + "type": "extension", + "extension_group": "embedding", + "addon": "aliyun_text_embedding", + "name": "aliyun_text_embedding", + "property": { + "api_key": "${env:ALIYUN_TEXT_EMBEDDING_API_KEY}", + "model": "text-embedding-v3" + } + }, + { + "type": "extension", + "extension_group": "vector_storage", + "addon": "aliyun_analyticdb_vector_storage", + "name": "aliyun_analyticdb_vector_storage", + "property": { + "alibaba_cloud_access_key_id": "${env:ALIBABA_CLOUD_ACCESS_KEY_ID}", + "alibaba_cloud_access_key_secret": "${env:ALIBABA_CLOUD_ACCESS_KEY_SECRET}", + "adbpg_instance_id": "${env:ALIYUN_ANALYTICDB_INSTANCE_ID}", + "adbpg_instance_region": "${env:ALIYUN_ANALYTICDB_INSTANCE_REGION}", + "adbpg_account": "${env:ALIYUN_ANALYTICDB_ACCOUNT}", + "adbpg_account_password": "${env:ALIYUN_ANALYTICDB_ACCOUNT_PASSWORD}", + "adbpg_namespace": "${env:ALIYUN_ANALYTICDB_NAMESPACE}", + "adbpg_namespace_password": "${env:ALIYUN_ANALYTICDB_NAMESPACE_PASSWORD}" + } + }, + { + "type": "extension", + "extension_group": "file_chunker", + "addon": "file_chunker", + "name": "file_chunker", + "property": {} + }, + { + "type": "extension", + "extension_group": "llama_index", + "addon": "llama_index_chat_engine", + "name": "llama_index", + "property": { + "greeting": "TEN Agent connected. How can I help you today?", + "chat_memory_token_limit": 3000 } - ] } - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "va_deepgram_openai_cartesia", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": false, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "asr", - "addon": "deepgram_asr_python", - "name": "deepgram_asr", - "property": { - "api_key": "${env:DEEPGRAM_API_KEY}", - "language": "en-US", - "model": "nova-2", - "sample_rate": "16000" - } - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt", - "name": "openai_chatgpt", - "property": { - "base_url": "", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "gpt-4o-mini", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "cartesia_tts", - "name": "cartesia_tts", - "property": { - "api_key": "${env:CARTESIA_API_KEY}", - "cartesia_version": "2024-06-10", - "model_id": "sonic-english", - "voice_id": "f9836c6e-a0bd-460e-9d3c-f7299fa60f94", - "sample_rate": "16000" - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector_python", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "asr", - "extension": "deepgram_asr" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "on_user_joined", - "dest": [ - { - "extension_group": "asr", - "extension": "deepgram_asr" - } - ] - }, - { - "name": "on_user_left", - "dest": [ - { - "extension_group": "asr", - "extension": "deepgram_asr" - } - ] - }, - { - "name": "on_connection_failure", - "dest": [ - { - "extension_group": "asr", - "extension": "deepgram_asr" - } - ] - } - ] - }, - { - "extension_group": "asr", - "extension": "deepgram_asr", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "cartesia_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "cartesia_tts" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "cartesia_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - } - ] - }, - { - "name": "va_deepgram_openai_fish", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": false, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" - } - }, - { - "type": "extension", - "extension_group": "asr", - "addon": "deepgram_asr_python", - "name": "deepgram_asr", - "property": { - "api_key": "${env:DEEPGRAM_API_KEY}", - "language": "en-US", - "model": "nova-2", - "sample_rate": "16000" - } - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt", - "name": "openai_chatgpt", - "property": { - "base_url": "", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "gpt-4o-mini", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "fish_audio_tts", - "name": "fish_audio_tts", - "property": { - "api_key": "${env:FISH_AUDIO_TTS_KEY}", - "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", - "optimize_streaming_latency": true, - "request_timeout_seconds": 30, - "base_url": "https://api.fish.audio" - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector_python", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "asr", - "extension": "deepgram_asr" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "on_user_joined", - "dest": [ - { - "extension_group": "asr", - "extension": "deepgram_asr" - } - ] - }, - { - "name": "on_user_left", - "dest": [ - { - "extension_group": "asr", - "extension": "deepgram_asr" - } - ] - }, - { - "name": "on_connection_failure", - "dest": [ - { - "extension_group": "asr", - "extension": "deepgram_asr" - } - ] - } - ] - }, - { - "extension_group": "asr", - "extension": "deepgram_asr", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "fish_audio_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "fish_audio_tts" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "fish_audio_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - } - ] - }, - { - "name": "va_minimax_v2v", - "auto_start": false, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "publish_audio": true, - "publish_data": true - } - }, - { - "type": "extension", - "extension_group": "agora_sess_ctrl", - "addon": "agora_sess_ctrl", - "name": "agora_sess_ctrl", - "property": { - "wait_for_eos": true - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "minimax_v2v_python", - "name": "minimax_v2v_python", - "property": { - "in_sample_rate": 16000, - "token": "${env:MINIMAX_TOKEN}" - } - }, - { - "type": "extension", - "extension_group": "message_collector", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "agora_sess_ctrl", - "extension": "agora_sess_ctrl" - } - ] - } - ] - }, - { - "extension_group": "agora_sess_ctrl", - "extension": "agora_sess_ctrl", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "llm", - "extension": "minimax_v2v_python" - } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector" + }, + { + "extension_group": "chat_transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + }, + { + "name": "file_chunk", + "dest": [ + { + "extension_group": "file_chunker", + "extension": "file_chunker" + }, + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + }, + { + "name": "file_chunked", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + }, + { + "name": "update_querying_collection", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ] + }, + { + "extension_group": "llama_index", + "extension": "llama_index", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "chat_transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + }, + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + }, + { + "name": "call_chat", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + } + ] + }, + { + "name": "embed", + "dest": [ + { + "extension_group": "embedding", + "extension": "aliyun_text_embedding" + } + ] + }, + { + "name": "query_vector", + "dest": [ + { + "extension_group": "vector_storage", + "extension": "aliyun_analyticdb_vector_storage" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "chat_transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "http_server", + "extension": "http_server", + "cmd": [ + { + "name": "file_chunk", + "dest": [ + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector" + } + ] + }, + { + "name": "update_querying_collection", + "dest": [ + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector" + } + ] + } + ] + }, + { + "extension_group": "file_chunker", + "extension": "file_chunker", + "cmd": [ + { + "name": "embed_batch", + "dest": [ + { + "extension_group": "embedding", + "extension": "aliyun_text_embedding" + } + ] + }, + { + "name": "create_collection", + "dest": [ + { + "extension_group": "vector_storage", + "extension": "aliyun_analyticdb_vector_storage" + } + ] + }, + { + "name": "upsert_vector", + "dest": [ + { + "extension_group": "vector_storage", + "extension": "aliyun_analyticdb_vector_storage" + } + ] + }, + { + "name": "file_chunked", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ] + } ] - } - ], - "cmd": [ - { - "name": "start_of_sentence", - "dest": [ - { - "extension_group": "llm", - "extension": "minimax_v2v_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "flush" + }, + { + "name": "va_simple_openai_v2v", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "subscribe_audio_sample_rate": 24000 } - ] + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "openai_v2v_python", + "name": "openai_v2v_python", + "property": { + "api_key": "${env:OPENAI_REALTIME_API_KEY}", + "temperature": 0.9, + "model": "gpt-4o-realtime-preview", + "max_tokens": 2048, + "voice": "alloy", + "language": "en-US", + "server_vad": true, + "dump": true, + "history": 10 + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "openai_v2v_python", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] } - } - ] - } - ] - }, - { - "extension_group": "llm", - "extension": "minimax_v2v_python", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "message_collector", - "extension": "message_collector" - } - ] - } - ], - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "message_collector", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "rtc", - "extension": "agora_rtc" - } - ] - } - ] - } - ] - }, - { - "name": "camera_va_openai_azure_rtm", - "auto_start": true, - "nodes": [ - { - "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "subscribe_video": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf", - "subscribe_video_pix_fmt": 4 - } - }, - { - "type": "extension", - "extension_group": "rtm", - "addon": "agora_rtm", - "name": "agora_rtm", - "property": { - "channel": "ten_agent_test", - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "user_id": "0", - "rtm_enabled": true, - "rtm_is_stream": false, - "rtm_presence_enabled": true, - "rtm_metadata_enabled": true, - "rtm_lock_enabled": true - } - }, - { - "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt_python", - "name": "openai_chatgpt", - "property": { - "base_url": "${env:OPENAI_API_BASE}", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "${env:OPENAI_MODEL}", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", - "max_memory_length": 10, - "enable_tools": true - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "azure_tts", - "name": "azure_tts", - "property": { - "azure_subscription_key": "${env:AZURE_TTS_KEY}", - "azure_subscription_region": "${env:AZURE_TTS_REGION}", - "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - }, - { - "type": "extension", - "extension_group": "agora_rtm_wrapper", - "addon": "agora_rtm_wrapper", - "name": "agora_rtm_wrapper" - } - ], - "connections": [ - { - "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "video_frame": [ - { - "name": "video_frame", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ], - "cmd": [ - { - "name": "on_user_joined", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - }, - { - "name": "on_user_left", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - }, - { - "name": "on_user_audio_track_state_changed", - "dest": [ - { - "extension_group": "agora_rtm_wrapper", - "extension": "agora_rtm_wrapper" - } ] - } - ] - }, - { - "extension": "agora_rtm", - "extension_group": "rtm", - "data": [ - { - "name": "rtm_message_event", - "dest": [ - { - "extension_group": "openai_chatgpt", - "extension": "chatgpt", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "text", - "conversion_mode": "from_original", - "original_path": "message" - }, - { - "path": "is_final", - "conversion_mode": "fixed_value", - "value": true + }, + { + "name": "va_openai_v2v", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "subscribe_audio_sample_rate": 24000 + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "openai_v2v_python", + "name": "openai_v2v_python", + "property": { + "api_key": "${env:OPENAI_REALTIME_API_KEY}", + "temperature": 0.9, + "model": "gpt-4o-realtime-preview", + "max_tokens": 2048, + "voice": "alloy", + "language": "en-US", + "server_vad": true, + "dump": true, + "history": 10 + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + }, + { + "type": "extension", + "extension_group": "tools", + "addon": "weatherapi_tool_python", + "name": "weatherapi_tool_python", + "property": { + "api_key": "${env:WEATHERAPI_API_KEY}" + } + }, + { + "type": "extension", + "extension_group": "tools", + "addon": "bingsearch_tool_python", + "name": "bingsearch_tool_python", + "property": { + "api_key": "${env:BING_API_KEY}" } - ] } - } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "tools", + "extension": "bingsearch_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "openai_v2v_python", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + }, + { + "name": "tool_call_get_current_weather", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + }, + { + "name": "tool_call_get_past_weather", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + }, + { + "name": "tool_call_get_future_weather", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + }, + { + "name": "tool_call_bing_search", + "dest": [ + { + "extension_group": "tools", + "extension": "bingsearch_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + } ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } + }, + { + "name": "va_simple_openai_v2v_fish", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "subscribe_audio_sample_rate": 24000, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "openai_v2v_python", + "name": "openai_v2v_python", + "property": { + "api_key": "${env:OPENAI_REALTIME_API_KEY}", + "temperature": 0.9, + "model": "gpt-4o-realtime-preview", + "max_tokens": 2048, + "audio_out": false, + "input_transcript": false, + "language": "en-US", + "server_vad": true, + "dump": true, + "history": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "fish_audio_tts", + "name": "fish_audio_tts", + "property": { + "api_key": "${env:FISH_AUDIO_TTS_KEY}", + "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", + "optimize_streaming_latency": true, + "request_timeout_seconds": 30, + "base_url": "https://api.fish.audio" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "openai_v2v_python", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "transcriber", + "extension": "message_collector" + }, + { + "extension_group": "tts", + "extension": "fish_audio_tts" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "fish_audio_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "fish_audio_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + } ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - } + }, + { + "name": "va_openai_v2v_fish", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "subscribe_audio_sample_rate": 24000, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "openai_v2v_python", + "name": "openai_v2v_python", + "property": { + "api_key": "${env:OPENAI_REALTIME_API_KEY}", + "temperature": 0.9, + "model": "gpt-4o-realtime-preview", + "max_tokens": 2048, + "audio_out": false, + "input_transcript": false, + "language": "en-US", + "server_vad": true, + "dump": true, + "history": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "fish_audio_tts", + "name": "fish_audio_tts", + "property": { + "api_key": "${env:FISH_AUDIO_TTS_KEY}", + "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", + "optimize_streaming_latency": true, + "request_timeout_seconds": 30, + "base_url": "https://api.fish.audio" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + }, + { + "type": "extension", + "extension_group": "tools", + "addon": "weatherapi_tool_python", + "name": "weatherapi_tool_python", + "property": { + "api_key": "${env:WEATHERAPI_API_KEY}" + } + }, + { + "type": "extension", + "extension_group": "tools", + "addon": "bingsearch_tool_python", + "name": "bingsearch_tool_python", + "property": { + "api_key": "${env:BING_API_KEY}" + } + } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "tools", + "extension": "bingsearch_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "openai_v2v_python", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "transcriber", + "extension": "message_collector" + }, + { + "extension_group": "tts", + "extension": "fish_audio_tts" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "fish_audio_tts" + } + ] + }, + { + "name": "tool_call_get_current_weather", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + }, + { + "name": "tool_call_get_past_weather", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + }, + { + "name": "tool_call_get_future_weather", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + }, + { + "name": "tool_call_bing_search", + "dest": [ + { + "extension_group": "tools", + "extension": "bingsearch_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "fish_audio_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + } ] - } - ] - }, - { - "extension_group": "tts", - "extension": "azure_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } + }, + { + "name": "va_openai_v2v_storage", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "subscribe_audio_sample_rate": 24000 + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "openai_v2v_python", + "name": "openai_v2v_python", + "property": { + "api_key": "${env:OPENAI_REALTIME_API_KEY}", + "temperature": 0.9, + "model": "gpt-4o-realtime-preview", + "max_tokens": 2048, + "voice": "alloy", + "language": "en-US", + "server_vad": true, + "dump": true, + "history": 10, + "enable_storage": true + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + }, + { + "type": "extension", + "extension_group": "tools", + "addon": "weatherapi_tool_python", + "name": "weatherapi_tool_python", + "property": { + "api_key": "${env:WEATHERAPI_API_KEY}" + } + }, + { + "type": "extension", + "extension_group": "tools", + "addon": "bingsearch_tool_python", + "name": "bingsearch_tool_python", + "property": { + "api_key": "${env:BING_API_KEY}" + } + }, + { + "type": "extension", + "extension_group": "context", + "addon": "tsdb_firestore", + "name": "tsdb_firestore", + "property": { + "credentials": { + "type": "service_account", + "project_id": "${env:FIRESTORE_PROJECT_ID}", + "private_key_id": "${env:FIRESTORE_PRIVATE_KEY_ID}", + "private_key": "${env:FIRESTORE_PRIVATE_KEY}", + "client_email": "${env:FIRESTORE_CLIENT_EMAIL}", + "client_id": "${env:FIRESTORE_CLIENT_ID}", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "${env:FIRESTORE_CERT_URL}", + "universe_domain": "googleapis.com" + }, + "channel_name": "ten_agent_test", + "collection_name": "llm_context" + } + } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "realtime", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension_group": "realtime", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "tools", + "extension": "bingsearch_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension_group": "realtime", + "extension": "openai_v2v_python" + } + ] + } + ] + }, + { + "extension_group": "realtime", + "extension": "openai_v2v_python", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "data": [ + { + "name": "append", + "dest": [ + { + "extension_group": "context", + "extension": "tsdb_firestore" + } + ] + }, + { + "name": "text_data", + "dest": [ + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + }, + { + "name": "retrieve", + "dest": [ + { + "extension_group": "context", + "extension": "tsdb_firestore" + } + ] + }, + { + "name": "tool_call_get_current_weather", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + }, + { + "name": "tool_call_get_past_weather", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + }, + { + "name": "tool_call_get_future_weather", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + }, + { + "name": "tool_call_bing_search", + "dest": [ + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "tool_call" + } + ] + } + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + } ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } + }, + { + "name": "va_deepgram_openai_cartesia", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": false, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "asr", + "addon": "deepgram_asr_python", + "name": "deepgram_asr", + "property": { + "api_key": "${env:DEEPGRAM_API_KEY}", + "language": "en-US", + "model": "nova-2", + "sample_rate": "16000" + } + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt", + "name": "openai_chatgpt", + "property": { + "base_url": "", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "gpt-4o-mini", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "cartesia_tts", + "name": "cartesia_tts", + "property": { + "api_key": "${env:CARTESIA_API_KEY}", + "cartesia_version": "2024-06-10", + "model_id": "sonic-english", + "voice_id": "f9836c6e-a0bd-460e-9d3c-f7299fa60f94", + "sample_rate": "16000" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + }, + { + "name": "on_connection_failure", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + } + ] + }, + { + "extension_group": "asr", + "extension": "deepgram_asr", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "cartesia_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "cartesia_tts" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "cartesia_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + } ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "agora_rtm_wrapper", - "extension": "agora_rtm_wrapper" - } + }, + { + "name": "va_deepgram_openai_fish", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": false, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "asr", + "addon": "deepgram_asr_python", + "name": "deepgram_asr", + "property": { + "api_key": "${env:DEEPGRAM_API_KEY}", + "language": "en-US", + "model": "nova-2", + "sample_rate": "16000" + } + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt", + "name": "openai_chatgpt", + "property": { + "base_url": "", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "gpt-4o-mini", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "fish_audio_tts", + "name": "fish_audio_tts", + "property": { + "api_key": "${env:FISH_AUDIO_TTS_KEY}", + "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", + "optimize_streaming_latency": true, + "request_timeout_seconds": 30, + "base_url": "https://api.fish.audio" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + }, + { + "name": "on_connection_failure", + "dest": [ + { + "extension_group": "asr", + "extension": "deepgram_asr" + } + ] + } + ] + }, + { + "extension_group": "asr", + "extension": "deepgram_asr", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "fish_audio_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "fish_audio_tts" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "fish_audio_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + } ] - } - ] - }, - { - "extension_group": "agora_rtm_wrapper", - "extension": "agora_rtm_wrapper", - "cmd": [ - { - "name": "publish", - "dest": [ - { - "extension_group": "rtm", - "extension": "agora_rtm" - } + }, + { + "name": "va_minimax_v2v", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true + } + }, + { + "type": "extension", + "extension_group": "agora_sess_ctrl", + "addon": "agora_sess_ctrl", + "name": "agora_sess_ctrl", + "property": { + "wait_for_eos": true + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "minimax_v2v_python", + "name": "minimax_v2v_python", + "property": { + "in_sample_rate": 16000, + "token": "${env:MINIMAX_TOKEN}" + } + }, + { + "type": "extension", + "extension_group": "message_collector", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "agora_sess_ctrl", + "extension": "agora_sess_ctrl" + } + ] + } + ] + }, + { + "extension_group": "agora_sess_ctrl", + "extension": "agora_sess_ctrl", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "llm", + "extension": "minimax_v2v_python" + } + ] + } + ], + "cmd": [ + { + "name": "start_of_sentence", + "dest": [ + { + "extension_group": "llm", + "extension": "minimax_v2v_python", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "flush" + } + ] + } + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "minimax_v2v_python", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "message_collector", + "extension": "message_collector" + } + ] + } + ], + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "message_collector", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + } ] - }, - { - "name": "set_presence_state", - "dest": [ - { - "extension_group": "rtm", - "extension": "agora_rtm" - } + }, + { + "name": "camera_va_openai_azure_rtm", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "subscribe_video": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf", + "subscribe_video_pix_fmt": 4 + } + }, + { + "type": "extension", + "extension_group": "rtm", + "addon": "agora_rtm", + "name": "agora_rtm", + "property": { + "channel": "ten_agent_test", + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "user_id": "0", + "rtm_enabled": true, + "rtm_is_stream": false, + "rtm_presence_enabled": true, + "rtm_metadata_enabled": true, + "rtm_lock_enabled": true + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt_python", + "name": "openai_chatgpt", + "property": { + "base_url": "${env:OPENAI_API_BASE}", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}", + "greeting": "TEN Agent connected. How can I help you today?", + "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", + "max_memory_length": 10, + "enable_tools": true + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", + "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + }, + { + "type": "extension", + "extension_group": "agora_rtm_wrapper", + "addon": "agora_rtm_wrapper", + "name": "agora_rtm_wrapper" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "video_frame": [ + { + "name": "video_frame", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + }, + { + "name": "on_user_audio_track_state_changed", + "dest": [ + { + "extension_group": "agora_rtm_wrapper", + "extension": "agora_rtm_wrapper" + } + ] + } + ] + }, + { + "extension": "agora_rtm", + "extension_group": "rtm", + "data": [ + { + "name": "rtm_message_event", + "dest": [ + { + "extension_group": "openai_chatgpt", + "extension": "chatgpt", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "text", + "conversion_mode": "from_original", + "original_path": "message" + }, + { + "path": "is_final", + "conversion_mode": "fixed_value", + "value": true + } + ] + } + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "agora_rtm_wrapper", + "extension": "agora_rtm_wrapper" + } + ] + } + ] + }, + { + "extension_group": "agora_rtm_wrapper", + "extension": "agora_rtm_wrapper", + "cmd": [ + { + "name": "publish", + "dest": [ + { + "extension_group": "rtm", + "extension": "agora_rtm" + } + ] + }, + { + "name": "set_presence_state", + "dest": [ + { + "extension_group": "rtm", + "extension": "agora_rtm" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + } ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } + }, + { + "name": "va_minimax_realtime", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true + } + }, + { + "type": "extension", + "extension_group": "agora_sess_ctrl", + "addon": "agora_sess_ctrl", + "name": "agora_sess_ctrl", + "property": { + "wait_for_eos": true + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "minimax_realtime", + "name": "minimax_realtime", + "property": { + "token": "${env:MINIMAX_TOKEN}" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "agora_sess_ctrl", + "extension": "agora_sess_ctrl" + } + ] + } + ] + }, + { + "extension_group": "agora_sess_ctrl", + "extension": "agora_sess_ctrl", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "llm", + "extension": "minimax_realtime" + } + ] + } + ], + "cmd": [ + { + "name": "start_of_sentence", + "dest": [ + { + "extension_group": "llm", + "extension": "minimax_realtime", + "msg_conversion": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "_ten.name", + "conversion_mode": "fixed_value", + "value": "flush" + } + ] + } + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "minimax_realtime", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + } ] - } - ] - } + } ] - } - ] - } -} + } +} \ No newline at end of file