diff --git a/.env.example b/.env.example index 0f4db19a..09ac4703 100644 --- a/.env.example +++ b/.env.example @@ -11,9 +11,6 @@ LOG_PATH=/tmp/astra LOG_STDOUT=true # Graph designer server port GRAPH_DESIGNER_SERVER_PORT=49483 -# The corresponding graph name based on the language -GRAPH_NAME_ZH=va.openai.azure -GRAPH_NAME_EN=va.openai.azure # Server port SERVER_PORT=8080 # Maximum number of workers diff --git a/agents/.gitignore b/agents/.gitignore index 2f785c62..19cdf170 100644 --- a/agents/.gitignore +++ b/agents/.gitignore @@ -27,4 +27,3 @@ xdump_config .vscode *.pyc *.pyc.* -/property.json diff --git a/agents/property.json b/agents/property.json new file mode 100644 index 00000000..808b159f --- /dev/null +++ b/agents/property.json @@ -0,0 +1,2538 @@ +{ + "_ten": { + "log_level": 3, + "predefined_graphs": [ + { + "name": "va.openai.azure", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt", + "name": "openai_chatgpt", + "property": { + "base_url": "", + "api_key": "", + "frequency_penalty": 0.9, + "model": "gpt-3.5-turbo", + "max_tokens": 512, + "prompt": "", + "proxy_url": "", + "greeting": "ASTRA agent connected. How can i help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "", + "azure_subscription_region": "", + "azure_synthesis_voice_name": "en-US-JaneNeural" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "chat_transcriber", + "name": "chat_transcriber" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "chatgpt" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "transcriber" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "cmd_conversions": [ + { + "cmd": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "is_final", + "type": "fixed_value", + "value": "bool(true)" + }, + { + "path": "stream_id", + "type": "fixed_value", + "value": "uint32(999)" + } + ] + } + } + ] + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + } + ] + }, + { + "name": "va.openai.11labs", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt", + "name": "openai_chatgpt", + "property": { + "base_url": "", + "api_key": "", + "frequency_penalty": 0.9, + "model": "gpt-3.5-turbo", + "max_tokens": 512, + "prompt": "", + "proxy_url": "", + "greeting": "ASTRA agent connected. How can i help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "elevenlabs_tts", + "name": "elevenlabs_tts", + "property": { + "api_key": "", + "model_id": "eleven_multilingual_v2", + "optimize_streaming_latency": 0, + "request_timeout_seconds": 30, + "similarity_boost": 0.75, + "speaker_boost": false, + "stability": 0.5, + "style": 0.0, + "voice_id": "pNInz6obpgDQGcFmaJgB" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "chat_transcriber", + "name": "chat_transcriber" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "chatgpt" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "transcriber" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "elevenlabs_tts" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "cmd_conversions": [ + { + "cmd": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "is_final", + "type": "fixed_value", + "value": "bool(true)" + }, + { + "path": "stream_id", + "type": "fixed_value", + "value": "uint32(999)" + } + ] + } + } + ] + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "elevenlabs_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "elevenlabs_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + } + ] + }, + { + "name": "va.bedrock.azure", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "bedrock", + "addon": "bedrock_llm_python", + "name": "bedrock_llm", + "property": { + "region": "us-east-1", + "access_key": "", + "secret_key": "", + "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "max_tokens": 512, + "prompt": "", + "greeting": "ASTRA agent connected. How can i help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "", + "azure_subscription_region": "", + "azure_synthesis_voice_name": "en-US-JaneNeural" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "bedrock" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] + } + ] + }, + { + "name": "va.openai.cosy", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt_python", + "name": "openai_chatgpt", + "property": { + "base_url": "", + "api_key": "", + "frequency_penalty": 0.9, + "model": "gpt-3.5-turbo", + "max_tokens": 512, + "prompt": "", + "proxy_url": "", + "greeting": "ASTRA agent connected. How can i help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "cosy_tts", + "name": "cosy_tts", + "property": { + "api_key": "", + "model": "cosyvoice-v1", + "voice": "longxiaochun", + "sample_rate": 16000 + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chat_transcriber", + "addon": "chat_transcriber_python", + "name": "chat_transcriber" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "chatgpt" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "chat_transcriber" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "cosy_tts" + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber", + "cmd_conversions": [ + { + "cmd": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "is_final", + "type": "fixed_value", + "value": "bool(true)" + }, + { + "path": "stream_id", + "type": "fixed_value", + "value": "uint32(999)" + } + ] + } + } + ] + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "cosy_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "cosy_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + } + ] + }, + { + "name": "va.qwen.cosy", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "qwen_llm_python", + "name": "qwen_llm", + "property": { + "api_key": "", + "model": "qwen-max", + "max_tokens": 512, + "prompt": "", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "cosy_tts", + "name": "cosy_tts", + "property": { + "api_key": "", + "model": "cosyvoice-v1", + "voice": "longxiaochun", + "sample_rate": 16000 + } + }, + { + "type": "extension", + "extension_group": "chat_transcriber", + "addon": "chat_transcriber_python", + "name": "chat_transcriber" + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "llm" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "chat_transcriber" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "qwen_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "cosy_tts" + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber", + "cmd_conversions": [ + { + "cmd": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "is_final", + "type": "fixed_value", + "value": "bool(true)" + }, + { + "path": "stream_id", + "type": "fixed_value", + "value": "uint32(999)" + } + ] + } + } + ] + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "cosy_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "cosy_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + } + ] + }, + { + "name": "va.bedrock.polly", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "bedrock", + "addon": "bedrock_llm_python", + "name": "bedrock_llm", + "property": { + "region": "us-east-1", + "access_key": "", + "secret_key": "", + "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "max_tokens": 512, + "prompt": "", + "greeting": "ASTRA agent connected. How can i help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "polly_tts", + "name": "polly_tts", + "property": { + "region": "us-east-1", + "access_key": "", + "secret_key": "", + "engine": "generative", + "voice": "Ruth", + "sample_rate": "16000", + "lang_code": "en-US" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "bedrock" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "polly_tts" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "polly_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "polly_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] + } + ] + }, + { + "name": "va.transcribe-bedrock.polly", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "baf537f77ebc4187a06a1628a1827f14", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": false, + "enable_agora_asr": false, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "asr", + "addon": "transcribe_asr", + "name": "transcribe_asr", + "property": { + "region": "us-east-1", + "access_key": "", + "secret_key": "", + "sample_rate": "16000", + "lang_code": "en-US" + } + }, + { + "type": "extension", + "extension_group": "bedrock", + "addon": "bedrock_llm_python", + "name": "bedrock_llm", + "property": { + "region": "us-east-1", + "access_key": "", + "secret_key": "", + "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "max_tokens": 512, + "prompt": "", + "greeting": "ASTRA agent connected. How can i help you today?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "polly_tts", + "name": "polly_tts", + "property": { + "region": "us-east-1", + "access_key": "", + "secret_key": "", + "engine": "generative", + "voice": "Ruth", + "sample_rate": "16000", + "lang_code": "en-US" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "bedrock" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "asr" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "asr", + "extension": "transcribe_asr" + } + ] + } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "asr", + "extension": "transcribe_asr" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "asr", + "extension": "transcribe_asr" + } + ] + }, + { + "name": "on_connection_failure", + "dest": [ + { + "extension_group": "asr", + "extension": "transcribe_asr" + } + ] + } + ] + }, + { + "extension_group": "asr", + "extension": "transcribe_asr", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] + }, + { + "extension_group": "bedrock", + "extension": "bedrock_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "polly_tts" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "polly_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "polly_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "bedrock", + "extension": "bedrock_llm" + } + ] + } + ] + } + ] + }, + { + "name": "camera.va.openai.azure.cn", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "subscribe_video": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "zh-CN", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf", + "subscribe_video_pix_fmt": 4 + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt_python", + "name": "openai_chatgpt", + "property": { + "base_url": "", + "api_key": "", + "frequency_penalty": 0.9, + "model": "gpt-4o", + "max_tokens": 512, + "prompt": "", + "proxy_url": "", + "greeting": "Astra已连接，需要我为您提供什么帮助?", + "checking_vision_text_items": "[\"让我看看你的摄像头...\",\"让我看一下...\",\"我看一下，请稍候...\"]", + "max_memory_length": 10, + "enable_tools": true + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "", + "azure_subscription_region": "", + "azure_synthesis_voice_name": "en-US-JaneNeural" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "chat_transcriber", + "name": "chat_transcriber" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "chatgpt" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "transcriber" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber" + } + ] + } + ], + "video_frame": [ + { + "name": "video_frame", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "cmd_conversions": [ + { + "cmd": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "is_final", + "type": "fixed_value", + "value": "bool(true)" + }, + { + "path": "stream_id", + "type": "fixed_value", + "value": "uint32(999)" + } + ] + } + } + ] + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + } + ] + }, + { + "name": "camera.va.openai.azure.en", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "subscribe_video": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf", + "subscribe_video_pix_fmt": 4 + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "chatgpt", + "addon": "openai_chatgpt_python", + "name": "openai_chatgpt", + "property": { + "base_url": "", + "api_key": "", + "frequency_penalty": 0.9, + "model": "gpt-4o-mini", + "max_tokens": 512, + "prompt": "", + "proxy_url": "", + "greeting": "ASTRA agent connected. How can i help you today?", + "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", + "max_memory_length": 10, + "enable_tools": true + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "", + "azure_subscription_region": "", + "azure_synthesis_voice_name": "en-US-JaneNeural" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "chat_transcriber", + "name": "chat_transcriber" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "chatgpt" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "transcriber" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber" + } + ] + } + ], + "video_frame": [ + { + "name": "video_frame", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + }, + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "cmd_conversions": [ + { + "cmd": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "is_final", + "type": "fixed_value", + "value": "bool(true)" + }, + { + "path": "stream_id", + "type": "fixed_value", + "value": "uint32(999)" + } + ] + } + } + ] + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "openai_chatgpt" + } + ] + } + ] + } + ] + }, + { + "name": "va.gemini.azure", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "gemini_llm_python", + "name": "gemini_llm", + "property": { + "api_key": "", + "greeting": "ASTRA agent connected. How can i help you today?", + "max_memory_length": 10, + "max_output_tokens": 512, + "model": "gemini-1.0-pro-latest", + "prompt": "", + "temperature": 0.1, + "top_k": 40, + "top_p": 0.95 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "", + "azure_subscription_region": "", + "azure_synthesis_voice_name": "en-US-JaneNeural" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "chat_transcriber", + "name": "chat_transcriber" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "default" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "llm" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "tts" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "transcriber" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "llm", + "extension": "gemini_llm" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber" + } + ] + } + ] + }, + { + "extension_group": "llm", + "extension": "gemini_llm", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "cmd_conversions": [ + { + "cmd": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "is_final", + "type": "fixed_value", + "value": "bool(true)" + }, + { + "path": "stream_id", + "type": "fixed_value", + "value": "uint32(999)" + } + ] + } + } + ] + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "chat_transcriber", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llm", + "extension": "gemini_llm" + } + ] + } + ] + } + ] + }, + { + "name": "va.qwen.rag", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "qwen_llm_python", + "name": "qwen_llm", + "property": { + "api_key": "", + "model": "qwen-max", + "max_tokens": 512, + "prompt": "", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "cosy_tts", + "name": "cosy_tts", + "property": { + "api_key": "", + "model": "cosyvoice-v1", + "voice": "longxiaochun", + "sample_rate": 16000 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "", + "azure_subscription_region": "", + "azure_synthesis_voice_name": "en-US-JaneNeural" + } + }, + { + "type": "extension", + "extension_group": "chat_transcriber", + "addon": "chat_transcriber_python", + "name": "chat_transcriber" + }, + { + "type": "extension", + "extension_group": "interrupt_detector", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "http_server", + "addon": "http_server_python", + "name": "http_server", + "property": { + "listen_addr": "127.0.0.1", + "listen_port": 8080 + } + }, + { + "type": "extension", + "extension_group": "embedding", + "addon": "aliyun_text_embedding", + "name": "aliyun_text_embedding", + "property": { + "api_key": "", + "model": "text-embedding-v3" + } + }, + { + "type": "extension", + "extension_group": "vector_storage", + "addon": "aliyun_analyticdb_vector_storage", + "name": "aliyun_analyticdb_vector_storage", + "property": { + "alibaba_cloud_access_key_id": "", + "alibaba_cloud_access_key_secret": "", + "adbpg_instance_id": "", + "adbpg_instance_region": "cn-shanghai", + "adbpg_account": "", + "adbpg_account_password": "", + "adbpg_namespace": "", + "adbpg_namespace_password": "" + } + }, + { + "type": "extension", + "extension_group": "file_chunker", + "addon": "file_chunker", + "name": "file_chunker", + "property": {} + }, + { + "type": "extension", + "extension_group": "llama_index", + "addon": "llama_index_chat_engine", + "name": "llama_index", + "property": { + "greeting": "ASTRA agent connected. How can i help you today?", + "chat_memory_token_limit": 3000 + } + } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector" + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber" + } + ] + } + ] + }, + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + }, + { + "name": "file_chunk", + "dest": [ + { + "extension_group": "file_chunker", + "extension": "file_chunker" + }, + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + }, + { + "name": "file_chunked", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + }, + { + "name": "update_querying_collection", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ] + }, + { + "extension_group": "llama_index", + "extension": "llama_index", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber", + "cmd_conversions": [ + { + "cmd": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "is_final", + "type": "fixed_value", + "value": "bool(true)" + }, + { + "path": "stream_id", + "type": "fixed_value", + "value": "uint32(999)" + } + ] + } + } + ] + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + }, + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + }, + { + "name": "call_chat", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + } + ] + }, + { + "name": "embed", + "dest": [ + { + "extension_group": "embedding", + "extension": "aliyun_text_embedding" + } + ] + }, + { + "name": "query_vector", + "dest": [ + { + "extension_group": "vector_storage", + "extension": "aliyun_analyticdb_vector_storage" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "http_server", + "extension": "http_server", + "cmd": [ + { + "name": "file_chunk", + "dest": [ + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector" + } + ] + }, + { + "name": "update_querying_collection", + "dest": [ + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector" + } + ] + } + ] + }, + { + "extension_group": "file_chunker", + "extension": "file_chunker", + "cmd": [ + { + "name": "embed_batch", + "dest": [ + { + "extension_group": "embedding", + "extension": "aliyun_text_embedding" + } + ] + }, + { + "name": "create_collection", + "dest": [ + { + "extension_group": "vector_storage", + "extension": "aliyun_analyticdb_vector_storage" + } + ] + }, + { + "name": "upsert_vector", + "dest": [ + { + "extension_group": "vector_storage", + "extension": "aliyun_analyticdb_vector_storage" + } + ] + }, + { + "name": "file_chunked", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/playground/src/app/layout.tsx b/playground/src/app/layout.tsx index 99aaff19..b6153573 100644 --- a/playground/src/app/layout.tsx +++ b/playground/src/app/layout.tsx @@ -7,7 +7,7 @@ import './global.css' export const metadata: Metadata = { title: "Astra.ai", - description: "Generated by create next app", + description: "A multimodal agent powered by TEN", appleWebApp: { capable: true, statusBarStyle: "black", diff --git a/playground/src/common/constant.ts b/playground/src/common/constant.ts index 81327d1e..2d8f809b 100644 --- a/playground/src/common/constant.ts +++ b/playground/src/common/constant.ts @@ -69,3 +69,45 @@ export const COLOR_LIST: ColorItem[] = [{ default: "#481C3F" }] +export type VoiceTypeMap = { + [voiceType: string]: string; +}; + +export type VendorNameMap = { + [vendorName: string]: VoiceTypeMap; +}; + +export type LanguageMap = { + [language: string]: VendorNameMap; +}; + +export const voiceNameMap: LanguageMap = { + "zh-CN": { + azure: { + male: "zh-CN-YunxiNeural", + female: "zh-CN-XiaoxiaoNeural", + }, + elevenlabs: { + male: "pNInz6obpgDQGcFmaJgB", // Adam + female: "Xb7hH8MSUJpSbSDYk0k2", // Alice + }, + polly: { + male: "Zhiyu", + female: "Zhiyu", + }, + }, + "en-US": { + azure: { + male: "en-US-BrianNeural", + female: "en-US-JaneNeural", + }, + elevenlabs: { + male: "pNInz6obpgDQGcFmaJgB", // Adam + female: "Xb7hH8MSUJpSbSDYk0k2", // Alice + }, + polly: { + male: "Matthew", + female: "Ruth", + }, + }, +}; \ No newline at end of file diff --git a/playground/src/common/graph.ts b/playground/src/common/graph.ts new file mode 100644 index 00000000..9540412f --- /dev/null +++ b/playground/src/common/graph.ts @@ -0,0 +1,46 @@ +import { voiceNameMap } from "./constant" + +export const getGraphWithLanguage = (graphName: string, language: string) => { + if (graphName == "camera.va.openai.azure") { + if (language == "zh-CN") { + return `${graphName}.cn` + } + return `${graphName}.en` + } + return graphName +} + +export const getGraphProperties = (graphName: string, language: string, voiceType: string) => { + if (graphName == "camera.va.openai.azure") { + return { + "agora_rtc": { + "agora_asr_language": language, + }, + "openai_chatgpt": { + "model": "gpt-4o" + }, + "azure_tts": { + "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] + } + } + } else if (graphName == "va.openai.azure") { + return { + "agora_rtc": { + "agora_asr_language": language, + }, + "azure_tts": { + "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] + } + } + } else if (graphName == "va.qwen.rag") { + return { + "agora_rtc": { + "agora_asr_language": language, + }, + "azure_tts": { + "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] + } + } + } + return {} +} \ No newline at end of file diff --git a/playground/src/common/index.ts b/playground/src/common/index.ts index 3c2b0300..48ffa71f 100644 --- a/playground/src/common/index.ts +++ b/playground/src/common/index.ts @@ -4,3 +4,4 @@ export * from "./utils" export * from "./storage" export * from "./request" export * from "./mock" +export * from "./graph" \ No newline at end of file diff --git a/playground/src/common/request.ts b/playground/src/common/request.ts index 6d6a6692..10803c6a 100644 --- a/playground/src/common/request.ts +++ b/playground/src/common/request.ts @@ -1,12 +1,12 @@ +import { AnyObject } from "antd/es/_util/type" import { REQUEST_URL } from "./constant" import { genUUID } from "./utils" interface StartRequestConfig { channel: string userId: number, - language: string - voiceType: string graphName: string + properties: AnyObject } interface GenAgoraDataConfig { @@ -35,15 +35,14 @@ export const apiGenAgoraData = async (config: GenAgoraDataConfig) => { export const apiStartService = async (config: StartRequestConfig): Promise => { const url = `${REQUEST_URL}/start` - const { language, channel, userId, voiceType, graphName } = config + const { channel, userId, graphName, properties } = config const data = { request_id: genUUID(), - agora_asr_language: language, channel_name: channel, openai_proxy_url: "", remote_stream_id: userId, - voice_type: voiceType, graph_name: graphName, + properties, } let resp: any = await fetch(url, { method: "POST", diff --git a/playground/src/common/utils.ts b/playground/src/common/utils.ts index d2a144f9..1d6f0d00 100644 --- a/playground/src/common/utils.ts +++ b/playground/src/common/utils.ts @@ -56,4 +56,4 @@ export const genUUID = () => { export const isMobile = () => { return /Mobile|iPhone|iPad|Android|Windows Phone/i.test(navigator.userAgent) -} +} \ No newline at end of file diff --git a/playground/src/platform/mobile/description/index.tsx b/playground/src/platform/mobile/description/index.tsx index 66425d8f..a3f6f472 100644 --- a/playground/src/platform/mobile/description/index.tsx +++ b/playground/src/platform/mobile/description/index.tsx @@ -1,7 +1,10 @@ import { setAgentConnected } from "@/store/reducers/global" import { DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, - apiStartService, apiStopService, REQUEST_URL + apiStartService, apiStopService, REQUEST_URL, + voiceNameMap, + getGraphWithLanguage, + getGraphProperties } from "@/common" import { message } from "antd" import { useEffect, useState } from "react" @@ -48,9 +51,8 @@ const Description = () => { const res = await apiStartService({ channel, userId, - language, - voiceType, - graphName, + graphName: getGraphWithLanguage(graphName, language), + properties: getGraphProperties(graphName, language, voiceType) }) const { code, msg } = res || {} if (code != 0) { diff --git a/playground/src/platform/pc/description/index.tsx b/playground/src/platform/pc/description/index.tsx index 2b395d9b..e26a0c0c 100644 --- a/playground/src/platform/pc/description/index.tsx +++ b/playground/src/platform/pc/description/index.tsx @@ -1,7 +1,10 @@ import { setAgentConnected } from "@/store/reducers/global" import { DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, - apiStartService, apiStopService, REQUEST_URL + apiStartService, apiStopService, REQUEST_URL, + voiceNameMap, + getGraphWithLanguage, + getGraphProperties } from "@/common" import { Select, Button, message, Upload } from "antd" import { useEffect, useState, MouseEventHandler } from "react" @@ -48,9 +51,8 @@ const Description = () => { const res = await apiStartService({ channel, userId, - language, - voiceType, - graphName + graphName: getGraphWithLanguage(graphName, language), + properties: getGraphProperties(graphName, language, voiceType) }) const { code, msg } = res || {} if (code != 0) { diff --git a/server/internal/config.go b/server/internal/config.go index ac5bf763..a873e3d5 100644 --- a/server/internal/config.go +++ b/server/internal/config.go @@ -2,7 +2,6 @@ package internal import ( "log/slog" - "os" ) type Prop struct { @@ -34,9 +33,6 @@ const ( PropertyJsonFile = "./agents/property.json" // Token expire time tokenExpirationInSeconds = uint32(86400) - // Voice type - voiceTypeMale = "male" - voiceTypeFemale = "female" ) var ( @@ -138,17 +134,8 @@ var ( }, } - // The corresponding graph name based on the language - graphNameMap = map[string]string{ - languageChinese: "va.openai.azure", - languageEnglish: "va.openai.azure", - } - // Retrieve parameters from the request and map them to the property.json file startPropMap = map[string][]Prop{ - "AgoraAsrLanguage": { - {ExtensionName: extensionNameAgoraRTC, Property: "agora_asr_language"}, - }, "ChannelName": { {ExtensionName: extensionNameAgoraRTC, Property: "channel"}, }, @@ -158,54 +145,8 @@ var ( "Token": { {ExtensionName: extensionNameAgoraRTC, Property: "token"}, }, - "VoiceType": { - {ExtensionName: extensionNameAzureTTS, Property: "azure_synthesis_voice_name"}, - {ExtensionName: extensionNameElevenlabsTTS, Property: "voice_id"}, - }, "WorkerHttpServerPort": { {ExtensionName: extensionNameHttpServer, Property: "listen_port"}, }, } - - // Map the voice name to the voice type - voiceNameMap = map[string]map[string]map[string]string{ - languageChinese: { - extensionNameAzureTTS: { - voiceTypeMale: "zh-CN-YunxiNeural", - voiceTypeFemale: "zh-CN-XiaoxiaoNeural", - }, - extensionNameElevenlabsTTS: { - voiceTypeMale: "pNInz6obpgDQGcFmaJgB", // Adam - voiceTypeFemale: "Xb7hH8MSUJpSbSDYk0k2", // Alice - }, - extensionNamePollyTTS: { - voiceTypeMale: "Zhiyu", - voiceTypeFemale: "Zhiyu", - }, - }, - languageEnglish: { - extensionNameAzureTTS: { - voiceTypeMale: "en-US-BrianNeural", - voiceTypeFemale: "en-US-JaneNeural", - }, - extensionNameElevenlabsTTS: { - voiceTypeMale: "pNInz6obpgDQGcFmaJgB", // Adam - voiceTypeFemale: "Xb7hH8MSUJpSbSDYk0k2", // Alice - }, - extensionNamePollyTTS: { - voiceTypeMale: "Matthew", - voiceTypeFemale: "Ruth", - }, - }, - } ) - -func SetGraphNameMap() { - if graphNameZH := os.Getenv("GRAPH_NAME_ZH"); graphNameZH != "" { - graphNameMap[languageChinese] = graphNameZH - } - - if graphNameEN := os.Getenv("GRAPH_NAME_EN"); graphNameEN != "" { - graphNameMap[languageEnglish] = graphNameEN - } -} diff --git a/server/internal/http_server.go b/server/internal/http_server.go index 34e6c080..b30fdb4c 100644 --- a/server/internal/http_server.go +++ b/server/internal/http_server.go @@ -22,6 +22,7 @@ import ( "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/gogf/gf/crypto/gmd5" + "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) @@ -48,14 +49,13 @@ type PingReq struct { } type StartReq struct { - RequestId string `json:"request_id,omitempty"` - AgoraAsrLanguage string `json:"agora_asr_language,omitempty"` - ChannelName string `json:"channel_name,omitempty"` - GraphName string `json:"graph_name,omitempty"` - RemoteStreamId uint32 `json:"remote_stream_id,omitempty"` - Token string `json:"token,omitempty"` - VoiceType string `json:"voice_type,omitempty"` - WorkerHttpServerPort int32 `json:"worker_http_server_port,omitempty"` + RequestId string `json:"request_id,omitempty"` + ChannelName string `json:"channel_name,omitempty"` + GraphName string `json:"graph_name,omitempty"` + RemoteStreamId uint32 `json:"remote_stream_id,omitempty"` + Token string `json:"token,omitempty"` + WorkerHttpServerPort int32 `json:"worker_http_server_port,omitempty"` + Properties map[string]map[string]interface{} `json:"properties,omitempty"` } type StopReq struct { @@ -369,14 +369,6 @@ func (s *HttpServer) processProperty(req *StartReq) (propertyJsonFile string, lo // Get graph name graphName := req.GraphName - language := req.AgoraAsrLanguage - if graphName == "camera.va.openai.azure" { - if language == languageChinese { - graphName = "camera.va.openai.azure.cn" - } else { - graphName = "camera.va.openai.azure.en" - } - } if graphName == "" { slog.Error("graph_name is mandatory", "requestId", req.RequestId, logTag) return @@ -393,16 +385,52 @@ func (s *HttpServer) processProperty(req *StartReq) (propertyJsonFile string, lo } graph := fmt.Sprintf(`_ten.predefined_graphs.#(name=="%s")`, graphName) + + // Get the array of graphs + graphs := gjson.Get(propertyJson, "_ten.predefined_graphs").Array() + + // Create a new array for graphs that match the name + var newGraphs []string + for _, graph := range graphs { + if graph.Get("name").String() == graphName { + newGraphs = append(newGraphs, graph.Raw) + } + } + + if len(newGraphs) == 0 { + slog.Error("handlerStart graph not found", "graph", graphName, "requestId", req.RequestId, logTag) + err = fmt.Errorf("graph not found") + return + } + + // Replace the predefined_graphs array with the filtered array + propertyJson, _ = sjson.SetRaw(propertyJson, "_ten.predefined_graphs", fmt.Sprintf("[%s]", strings.Join(newGraphs, ","))) + // Automatically start on launch propertyJson, _ = sjson.Set(propertyJson, fmt.Sprintf(`%s.auto_start`, graph), true) - // Set parameters from the request to property.json + // Set environment variable values to property.json + for envKey, envProps := range EnvPropMap { + if envVal := os.Getenv(envKey); envVal != "" { + for _, envProp := range envProps { + propertyJson, _ = sjson.Set(propertyJson, fmt.Sprintf(`%s.nodes.#(name=="%s").property.%s`, graph, envProp.ExtensionName, envProp.Property), envVal) + } + } + } + + // Set additional properties to property.json + for extensionName, props := range req.Properties { + if extKey := extensionName; extKey != "" { + for prop, val := range props { + propertyJson, _ = sjson.Set(propertyJson, fmt.Sprintf(`%s.nodes.#(name=="%s").property.%s`, graph, extKey, prop), val) + } + } + } + + // Set start parameters to property.json for key, props := range startPropMap { if val := getFieldValue(req, key); val != "" { for _, prop := range props { - if key == "VoiceType" { - val = voiceNameMap[req.AgoraAsrLanguage][prop.ExtensionName][req.VoiceType] - } propertyJson, _ = sjson.Set(propertyJson, fmt.Sprintf(`%s.nodes.#(name=="%s").property.%s`, graph, prop.ExtensionName, prop.Property), val) } } diff --git a/server/internal/worker.go b/server/internal/worker.go index d90452e1..51789d33 100644 --- a/server/internal/worker.go +++ b/server/internal/worker.go @@ -114,6 +114,9 @@ func (w *Worker) start(req *StartReq) (err error) { shell := fmt.Sprintf("cd /app/agents && %s --property %s", workerExec, w.PropertyJsonFile) slog.Info("Worker start", "requestId", req.RequestId, "shell", shell, logTag) cmd := exec.Command("sh", "-c", shell) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, // Start a new process group + } var stdoutWriter, stderrWriter io.Writer var logFile *os.File @@ -183,9 +186,9 @@ func (w *Worker) start(req *StartReq) (err error) { } func (w *Worker) stop(requestId string, channelName string) (err error) { - slog.Info("Worker stop start", "channelName", channelName, "requestId", requestId, logTag) + slog.Info("Worker stop start", "channelName", channelName, "requestId", requestId, "pid", w.Pid, logTag) - err = syscall.Kill(w.Pid, syscall.SIGTERM) + err = syscall.Kill(-w.Pid, syscall.SIGTERM) if err != nil { slog.Error("Worker kill failed", "err", err, "channelName", channelName, "worker", w, "requestId", requestId, logTag) return diff --git a/server/main.go b/server/main.go index 188ce196..80dcd95d 100644 --- a/server/main.go +++ b/server/main.go @@ -1,14 +1,11 @@ package main import ( - "fmt" "log/slog" "os" "strconv" "github.com/joho/godotenv" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" "app/internal" ) @@ -54,15 +51,6 @@ func main() { os.Exit(1) } - // Set graph name map - internal.SetGraphNameMap() - - // Process property.json - if err = processProperty(internal.PropertyJsonFile); err != nil { - slog.Error("process property.json failed", "err", err) - os.Exit(1) - } - // Start server httpServerConfig := &internal.HttpServerConfig{ AppId: agoraAppId, @@ -76,30 +64,3 @@ func main() { httpServer := internal.NewHttpServer(httpServerConfig) httpServer.Start() } - -func processProperty(propertyJsonFile string) (err error) { - content, err := os.ReadFile(propertyJsonFile) - if err != nil { - slog.Error("read property.json failed", "err", err, "propertyJsonFile", propertyJsonFile) - return - } - - propertyJson := string(content) - for i := range gjson.Get(propertyJson, "_ten.predefined_graphs").Array() { - graph := fmt.Sprintf("_ten.predefined_graphs.%d", i) - // Shut down all auto-starting Graphs - propertyJson, _ = sjson.Set(propertyJson, fmt.Sprintf(`%s.auto_start`, graph), false) - - // Set environment variable values to property.json - for envKey, envProps := range internal.EnvPropMap { - if envVal := os.Getenv(envKey); envVal != "" { - for _, envProp := range envProps { - propertyJson, _ = sjson.Set(propertyJson, fmt.Sprintf(`%s.nodes.#(name=="%s").property.%s`, graph, envProp.ExtensionName, envProp.Property), envVal) - } - } - } - } - - err = os.WriteFile(propertyJsonFile, []byte(propertyJson), 0644) - return -}