diff --git a/agents/bin/start b/agents/bin/start index 4b75df21..89b66cf9 100755 --- a/agents/bin/start +++ b/agents/bin/start @@ -4,6 +4,8 @@ set -e cd "$(dirname "${BASH_SOURCE[0]}")/.." +#export TEN_ENABLE_PYTHON_DEBUG=true +#export TEN_PYTHON_DEBUG_PORT=5678 export PYTHONPATH=$(pwd)/ten_packages/system/ten_ai_base/interface:$PYTHONPATH export LD_LIBRARY_PATH=$(pwd)/ten_packages/system/agora_rtc_sdk/lib:$(pwd)/ten_packages/extension/agora_rtm/lib:$(pwd)/ten_packages/system/azure_speech_sdk/lib diff --git a/agents/examples/experimental/property.json b/agents/examples/experimental/property.json index da615e9e..937be596 100644 --- a/agents/examples/experimental/property.json +++ b/agents/examples/experimental/property.json @@ -3797,6 +3797,201 @@ ] } ] + }, + { + "name": "va_coze_azure", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "extension_group": "default", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "default", + "addon": "interrupt_detector", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "glue", + "addon": "coze_python_async", + "name": "coze_python_async", + "property": { + "token": "", + "bot_id": "", + "base_url": "https://api.coze.cn", + "prompt": "", + "greeting": "TEN Agent connected. How can I help you today?" + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", + "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + } + }, + { + "type": "extension", + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" + } + ], + "connections": [ + { + "extension_group": "default", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "glue", + "extension": "coze_python_async" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "glue", + "extension": "coze_python_async" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "glue", + "extension": "coze_python_async" + } + ] + } + ] + }, + { + "extension_group": "glue", + "extension": "coze_python_async", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "tts", + "extension": "azure_tts" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "azure_tts", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "default", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "glue", + "extension": "coze_python_async" + } + ] + } + ] + } + ] } ] } diff --git a/agents/ten_packages/extension/coze_python_async/BUILD.gn b/agents/ten_packages/extension/coze_python_async/BUILD.gn new file mode 100644 index 00000000..05054fcf --- /dev/null +++ b/agents/ten_packages/extension/coze_python_async/BUILD.gn @@ -0,0 +1,20 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import("//build/feature/ten_package.gni") + +ten_package("coze_python_async") { + package_kind = "extension" + + resources = [ + "__init__.py", + "addon.py", + "extension.py", + "log.py", + "manifest.json", + "property.json", + "tests", + ] +} diff --git a/agents/ten_packages/extension/coze_python_async/README.md b/agents/ten_packages/extension/coze_python_async/README.md new file mode 100644 index 00000000..16e3fd21 --- /dev/null +++ b/agents/ten_packages/extension/coze_python_async/README.md @@ -0,0 +1,37 @@ +# coze_python_async + +This is a python extension for coze service. The schema of coze service is attached in `schema.yml`. + +An example of OpenAI wrapper is also attached in `examples/openai_wrapper.py`. + +## Features + +The extension will record history with count of `max_history`. + +- `api_url` (must have): the url for the coze service. +- `token` (must have): use Bearer token to support default auth + +The extension support flush that will close the existing http session. + +## API + +Refer to `api` definition in [manifest.json] and default values in [property.json](property.json). + +- In: + - `text_data` [data]: the asr result + - `flush` [cmd]: the flush signal +- Out: + - `flush` [cmd]: the flush signal + +## Examples + +You can run example using following command, and the wrapper service will listen 8000 by default. + +``` +> export API_TOKEN="xxx" && export OPENAI_API_KEY="xxx" && python3 openai_wrapper.py + +INFO: Started server process [162886] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +``` diff --git a/agents/ten_packages/extension/coze_python_async/__init__.py b/agents/ten_packages/extension/coze_python_async/__init__.py new file mode 100644 index 00000000..c22fdd7c --- /dev/null +++ b/agents/ten_packages/extension/coze_python_async/__init__.py @@ -0,0 +1,7 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from . import addon + diff --git a/agents/ten_packages/extension/coze_python_async/addon.py b/agents/ten_packages/extension/coze_python_async/addon.py new file mode 100644 index 00000000..364777f7 --- /dev/null +++ b/agents/ten_packages/extension/coze_python_async/addon.py @@ -0,0 +1,19 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import ( + Addon, + register_addon_as_extension, + TenEnv, +) + + +@register_addon_as_extension("coze_python_async") +class AsyncCozeExtensionAddon(Addon): + + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + from .extension import AsyncCozeExtension + ten_env.log_info("AsyncCozeExtensionAddon on_create_instance") + ten_env.on_create_instance_done(AsyncCozeExtension(name), context) diff --git a/agents/ten_packages/extension/coze_python_async/extension.py b/agents/ten_packages/extension/coze_python_async/extension.py new file mode 100644 index 00000000..53f674c6 --- /dev/null +++ b/agents/ten_packages/extension/coze_python_async/extension.py @@ -0,0 +1,322 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import asyncio +import traceback +import aiohttp +import json +import copy + +from typing import List, Any, AsyncGenerator +from dataclasses import dataclass + +from cozepy import ChatEventType, Message, TokenAuth, AsyncCoze, ChatEvent, Chat + +from ten import ( + AudioFrame, + VideoFrame, + AsyncTenEnv, + Cmd, + StatusCode, + CmdResult, + Data, +) + +from ten_ai_base import BaseConfig, ChatMemory +from ten_ai_base.llm import AsyncLLMBaseExtension, LLMCallCompletionArgs, LLMDataCompletionArgs, LLMToolMetadata +from ten_ai_base.types import LLMChatCompletionUserMessageParam, LLMToolResult + +CMD_IN_FLUSH = "flush" +CMD_IN_ON_USER_JOINED = "on_user_joined" +CMD_IN_ON_USER_LEFT = "on_user_left" +CMD_OUT_FLUSH = "flush" +CMD_OUT_TOOL_CALL = "tool_call" + +DATA_IN_TEXT_DATA_PROPERTY_IS_FINAL = "is_final" +DATA_IN_TEXT_DATA_PROPERTY_TEXT = "text" + +DATA_OUT_TEXT_DATA_PROPERTY_TEXT = "text" +DATA_OUT_TEXT_DATA_PROPERTY_END_OF_SEGMENT = "end_of_segment" + +CMD_PROPERTY_RESULT = "tool_result" + +def is_punctuation(char): + if char in [",", "，", ".", "。", "?", "？", "!", "！"]: + return True + return False + +def parse_sentences(sentence_fragment, content): + sentences = [] + current_sentence = sentence_fragment + for char in content: + current_sentence += char + if is_punctuation(char): + stripped_sentence = current_sentence + if any(c.isalnum() for c in stripped_sentence): + sentences.append(stripped_sentence) + current_sentence = "" + + remain = current_sentence + return sentences, remain + +@dataclass +class CozeConfig(BaseConfig): + base_url: str = "https://api.acoze.com" + bot_id: str = "" + token: str = "" + user_id: str = "TenAgent" + greeting: str = "" + max_history: int = 32 + +class AsyncCozeExtension(AsyncLLMBaseExtension): + config : CozeConfig = None + sentence_fragment: str = "" + ten_env: AsyncTenEnv = None + loop: asyncio.AbstractEventLoop = None + stopped: bool = False + users_count = 0 + memory: ChatMemory = None + + acoze: AsyncCoze = None + #conversation: str = "" + + async def on_init(self, ten_env: AsyncTenEnv) -> None: + await super().on_init(ten_env) + ten_env.log_debug("on_init") + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + await super().on_start(ten_env) + ten_env.log_debug("on_start") + + self.loop = asyncio.get_event_loop() + + self.config = CozeConfig.create(ten_env=ten_env) + ten_env.log_info(f"config: {self.config}") + + if not self.config.bot_id or not self.config.token: + ten_env.log_error("Missing required configuration") + return + + self.memory = ChatMemory(self.config.max_history) + try: + self.acoze = AsyncCoze(auth=TokenAuth(token=self.config.token), base_url=self.config.base_url) + ''' + self.conversation = await self.acoze.conversations.create(messages = [ + Message.build_user_question_text(self.config.prompt) + ] if self.config.prompt else []) + ''' + except Exception as e: + ten_env.log_error(f"Failed to create conversation {e}") + + self.ten_env = ten_env + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + await super().on_stop(ten_env) + ten_env.log_debug("on_stop") + + self.stopped = True + await self.queue.put(None) + + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: + await super().on_deinit(ten_env) + ten_env.log_debug("on_deinit") + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + cmd_name = cmd.get_name() + ten_env.log_debug("on_cmd name {}".format(cmd_name)) + + status = StatusCode.OK + detail = "success" + + if cmd_name == CMD_IN_FLUSH: + await self.flush_input_items(ten_env) + await ten_env.send_cmd(Cmd.create(CMD_OUT_FLUSH)) + ten_env.log_info("on flush") + elif cmd_name == CMD_IN_ON_USER_JOINED: + self.users_count += 1 + # Send greeting when first user joined + if self.config.greeting and self.users_count == 1: + self.send_text_output(ten_env, self.config.greeting, True) + elif cmd_name == CMD_IN_ON_USER_LEFT: + self.users_count -= 1 + else: + await super().on_cmd(ten_env, cmd) + return + + cmd_result = CmdResult.create(status) + cmd_result.set_property_string("detail", detail) + ten_env.return_result(cmd_result, cmd) + + async def on_call_chat_completion(self, ten_env: AsyncTenEnv, **kargs: LLMCallCompletionArgs) -> any: + raise Exception("Not implemented") + + async def on_data_chat_completion(self, ten_env: AsyncTenEnv, **kargs: LLMDataCompletionArgs) -> None: + if not self.acoze: + await self._send_text("Coze is not connected. Please check your configuration.", True) + return + + input: LLMChatCompletionUserMessageParam = kargs.get("messages", []) + messages = copy.copy(self.memory.get()) + if not input: + ten_env.log_warn("No message in data") + else: + messages.extend(input) + for i in input: + self.memory.put(i) + + total_output = "" + sentence_fragment = "" + calls = {} + + sentences = [] + self.ten_env.log_info(f"messages: {messages}") + response = self._stream_chat(messages=messages) + async for message in response: + self.ten_env.log_info(f"content: {message}") + try: + if message.event == ChatEventType.CONVERSATION_MESSAGE_DELTA: + total_output += message.message.content + sentences, sentence_fragment = parse_sentences( + sentence_fragment, message.message.content) + for s in sentences: + await self._send_text(s, False) + elif message.event == ChatEventType.CONVERSATION_MESSAGE_COMPLETED: + if sentence_fragment: + await self._send_text(sentence_fragment, True) + else: + await self._send_text("", True) + elif message.event == ChatEventType.CONVERSATION_CHAT_FAILED: + last_error = message.chat.last_error + if last_error and last_error.code == 4011: + await self._send_text("The Coze token has been depleted. Please check your token usage.", True) + else: + await self._send_text(last_error.msg, True) + except Exception as e: + self.ten_env.log_error(f"Failed to parse response: {message} {e}") + traceback.print_exc() + + self.memory.put({"role": "assistant", "content": total_output}) + self.ten_env.log_info(f"total_output: {total_output} {calls}") + + async def on_tools_update(self, ten_env: AsyncTenEnv, tool: LLMToolMetadata) -> None: + # Implement the logic for tool updates + return await super().on_tools_update(ten_env, tool) + + async def on_data(self, ten_env: AsyncTenEnv, data: Data) -> None: + data_name = data.get_name() + ten_env.log_info("on_data name {}".format(data_name)) + + is_final = False + input_text = "" + try: + is_final = data.get_property_bool(DATA_IN_TEXT_DATA_PROPERTY_IS_FINAL) + except Exception as err: + ten_env.log_info(f"GetProperty optional {DATA_IN_TEXT_DATA_PROPERTY_IS_FINAL} failed, err: {err}") + + try: + input_text = data.get_property_string(DATA_IN_TEXT_DATA_PROPERTY_TEXT) + except Exception as err: + ten_env.log_info(f"GetProperty optional {DATA_IN_TEXT_DATA_PROPERTY_TEXT} failed, err: {err}") + + if not is_final: + ten_env.log_info("ignore non-final input") + return + if not input_text: + ten_env.log_info("ignore empty text") + return + + ten_env.log_info(f"OnData input text: [{input_text}]") + + # Start an asynchronous task for handling chat completion + message = LLMChatCompletionUserMessageParam( + role="user", content=input_text) + await self.queue_input_item(False, messages=[message]) + + async def on_audio_frame(self, ten_env: AsyncTenEnv, audio_frame: AudioFrame) -> None: + pass + + async def on_video_frame(self, ten_env: AsyncTenEnv, video_frame: VideoFrame) -> None: + pass + + async def _send_text(self, text: str, end_of_segment: bool) -> None: + data = Data.create("text_data") + data.set_property_string(DATA_OUT_TEXT_DATA_PROPERTY_TEXT, text) + data.set_property_bool(DATA_OUT_TEXT_DATA_PROPERTY_END_OF_SEGMENT, end_of_segment) + self.ten_env.send_data(data) + + async def _stream_chat(self, messages: List[Any]) -> AsyncGenerator[ChatEvent, None]: + additionals = [] + for m in messages: + if m["role"] == "user": + additionals.append(Message.build_user_question_text(m["content"]).model_dump()) + elif m["role"] == "assistant": + additionals.append(Message.build_assistant_answer(m["content"]).model_dump()) + + def chat_stream_handler(event:str, event_data:Any) -> ChatEvent: + if event == ChatEventType.DONE: + raise StopAsyncIteration + elif event == ChatEventType.ERROR: + raise Exception(f"error event: {event_data}") # TODO: error struct format + elif event in [ + ChatEventType.CONVERSATION_MESSAGE_DELTA, + ChatEventType.CONVERSATION_MESSAGE_COMPLETED, + ]: + return ChatEvent(event=event, message=Message.model_validate_json(event_data)) + elif event in [ + ChatEventType.CONVERSATION_CHAT_CREATED, + ChatEventType.CONVERSATION_CHAT_IN_PROGRESS, + ChatEventType.CONVERSATION_CHAT_COMPLETED, + ChatEventType.CONVERSATION_CHAT_FAILED, + ChatEventType.CONVERSATION_CHAT_REQUIRES_ACTION, + ]: + return ChatEvent(event=event, chat=Chat.model_validate_json(event_data)) + else: + raise ValueError(f"invalid chat.event: {event}, {event_data}") + + async with aiohttp.ClientSession() as session: + try: + url = f"{self.config.base_url}/v3/chat" + headers = { + "Authorization": f"Bearer {self.config.token}", + } + params = { + "bot_id": self.config.bot_id, + "user_id": self.config.user_id, + "additional_messages": additionals, + "stream": True, + "auto_save_history": True, + #"conversation_id": self.conversation.id + } + event = "" + async with session.post(url, json=params, headers=headers) as response: + async for line in response.content: + if line: + try: + self.ten_env.log_info(f"line: {line}") + decoded_line = line.decode('utf-8').strip() + if decoded_line: + if decoded_line.startswith("data:"): + data = decoded_line[5:].strip() + yield chat_stream_handler(event=event, event_data=data.strip()) + elif decoded_line.startswith("event:"): + event = decoded_line[6:] + self.ten_env.log_info(f"event: {event}") + if event == "done": + break + else: + result = json.loads(decoded_line) + code = result.get("code", 0) + if code == 4000: + await self._send_text("Coze bot is not published.", True) + else: + self.ten_env.log_error(f"Failed to stream chat: {result['code']}") + await self._send_text("Coze bot is not connected. Please check your configuration.", True) + except Exception as e: + self.ten_env.log_error(f"Failed to stream chat: {e}") + except Exception as e: + traceback.print_exc() + self.ten_env.log_error(f"Failed to stream chat: {e}") + finally: + await session.close() diff --git a/agents/ten_packages/extension/coze_python_async/manifest.json b/agents/ten_packages/extension/coze_python_async/manifest.json new file mode 100644 index 00000000..faa66c06 --- /dev/null +++ b/agents/ten_packages/extension/coze_python_async/manifest.json @@ -0,0 +1,75 @@ +{ + "type": "extension", + "name": "coze_python_async", + "version": "0.3.1", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.4" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "BUILD.gn", + "**.tent", + "**.py", + "README.md", + "tests/**" + ] + }, + "api": { + "property": { + "base_url": { + "type": "string" + }, + "bot_id": { + "type": "string" + }, + "token": { + "type": "string" + }, + "user_id": { + "type": "string" + }, + "prompt": { + "type": "string" + }, + "greeting": { + "type": "string" + } + }, + "data_in": [ + { + "name": "text_data", + "property": { + "text": { + "type": "string" + } + } + } + ], + "data_out": [ + { + "name": "text_data", + "property": { + "text": { + "type": "string" + } + } + } + ], + "cmd_in": [ + { + "name": "flush" + } + ], + "cmd_out": [ + { + "name": "flush" + } + ] + } +} \ No newline at end of file diff --git a/agents/ten_packages/extension/coze_python_async/property.json b/agents/ten_packages/extension/coze_python_async/property.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/agents/ten_packages/extension/coze_python_async/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/agents/ten_packages/extension/coze_python_async/requirements.txt b/agents/ten_packages/extension/coze_python_async/requirements.txt new file mode 100644 index 00000000..be48f5e7 --- /dev/null +++ b/agents/ten_packages/extension/coze_python_async/requirements.txt @@ -0,0 +1 @@ +cozepy==0.6.2 \ No newline at end of file diff --git a/demo/src/app/api/agents/start/graph.tsx b/demo/src/app/api/agents/start/graph.ts similarity index 92% rename from demo/src/app/api/agents/start/graph.tsx rename to demo/src/app/api/agents/start/graph.ts index fba485e1..cff387b1 100644 --- a/demo/src/app/api/agents/start/graph.tsx +++ b/demo/src/app/api/agents/start/graph.ts @@ -96,7 +96,20 @@ export const getGraphProperties = ( "agora_asr_language": language, }, "openai_chatgpt": { - "model": "gpt-4o", + ...localizationOptions, + "prompt": prompt, + "greeting": greeting, + }, + "azure_tts": { + "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] + } + } + } else if (graphName == "va_coze_azure") { + return { + "agora_rtc": { + "agora_asr_language": language, + }, + "coze_python_async": { ...localizationOptions, "prompt": prompt, "greeting": greeting, diff --git a/demo/src/app/api/agents/start/route.tsx b/demo/src/app/api/agents/start/route.ts similarity index 80% rename from demo/src/app/api/agents/start/route.tsx rename to demo/src/app/api/agents/start/route.ts index ab98ceab..c12fbe9d 100644 --- a/demo/src/app/api/agents/start/route.tsx +++ b/demo/src/app/api/agents/start/route.ts @@ -26,15 +26,25 @@ export async function POST(request: NextRequest) { voice_type, prompt, greeting, + coze_token, + coze_bot_id, + coze_base_url, } = body; + let properties: any = getGraphProperties(graph_name, language, voice_type, prompt, greeting); + if (graph_name.includes("coze")) { + properties["coze_python_async"]["token"] = coze_token; + properties["coze_python_async"]["bot_id"] = coze_bot_id; + properties["coze_python_async"]["base_url"] = coze_base_url; + } + console.log(`Starting agent for request ID: ${JSON.stringify({ request_id, channel_name, user_uid, graph_name, // Get the graph properties based on the graph name, language, and voice type - properties: getGraphProperties(graph_name, language, voice_type, prompt, greeting), + properties, })}`); console.log(`AGENT_SERVER_URL: ${AGENT_SERVER_URL}/start`); @@ -46,7 +56,7 @@ export async function POST(request: NextRequest) { user_uid, graph_name, // Get the graph properties based on the graph name, language, and voice type - properties: getGraphProperties(graph_name, language, voice_type, prompt, greeting), + properties, }); const responseData = response.data; diff --git a/demo/src/app/layout.tsx b/demo/src/app/layout.tsx index c9fec642..b50f5576 100644 --- a/demo/src/app/layout.tsx +++ b/demo/src/app/layout.tsx @@ -41,7 +41,7 @@ export default function RootLayout({ > */} {children} {/* */} - + ) diff --git a/demo/src/common/constant.ts b/demo/src/common/constant.ts index a877bab8..1818bc0d 100644 --- a/demo/src/common/constant.ts +++ b/demo/src/common/constant.ts @@ -4,10 +4,12 @@ import { LanguageOptionItem, VoiceOptionItem, GraphOptionItem, + ICozeSettings, } from "@/types" export const GITHUB_URL = "https://github.com/TEN-framework/TEN-Agent" export const OPTIONS_KEY = "__options__" export const AGENT_SETTINGS_KEY = "__agent_settings__" +export const COZE_SETTINGS_KEY = "__coze_settings__" export const DEFAULT_OPTIONS: IOptions = { channel: "", userName: "", @@ -21,6 +23,17 @@ export const DEFAULT_AGENT_SETTINGS = { prompt: "", } +export enum ECozeBaseUrl { + // CN = "https://api.coze.cn", + GLOBAL = "https://api.coze.com", +} + +export const DEFAULT_COZE_SETTINGS: ICozeSettings = { + token: "", + bot_id: "", + base_url: ECozeBaseUrl.GLOBAL, +} + export const DESCRIPTION = "The World's First Multimodal AI Agent with the OpenAI Realtime API (Beta)" export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [ @@ -66,6 +79,10 @@ export const GRAPH_OPTIONS: GraphOptionItem[] = [ label: "Voice Agent with Vision - OpenAI LLM + Azure TTS + RTM", value: "camera_va_openai_azure_rtm", }, + { + label: "Voice Agent Coze Bot + Azure TTS", + value: "va_coze_azure", + }, ] export const isRagGraph = (graphName: string) => { diff --git a/demo/src/common/request.ts b/demo/src/common/request.ts index 6f6683b9..d6220c94 100644 --- a/demo/src/common/request.ts +++ b/demo/src/common/request.ts @@ -2,14 +2,17 @@ import { genUUID } from "./utils" import { Language } from "@/types" import axios from "axios" -interface StartRequestConfig { +export interface StartRequestConfig { channel: string - userId: number, - graphName: string, - language: Language, + userId: number + graphName: string + language: Language voiceType: "male" | "female" - prompt?: string, - greeting?: string, + prompt?: string + greeting?: string + coze_token?: string + coze_bot_id?: string + coze_base_url?: string } interface GenAgoraDataConfig { @@ -24,17 +27,30 @@ export const apiGenAgoraData = async (config: GenAgoraDataConfig) => { const data = { request_id: genUUID(), uid: userId, - channel_name: channel + channel_name: channel, } let resp: any = await axios.post(url, data) - resp = (resp.data) || {} + resp = resp.data || {} return resp } -export const apiStartService = async (config: StartRequestConfig): Promise => { +export const apiStartService = async ( + config: StartRequestConfig, +): Promise => { // look at app/api/agents/start/route.tsx for the server-side implementation const url = `/api/agents/start` - const { channel, userId, graphName, language, voiceType, greeting, prompt } = config + const { + channel, + userId, + graphName, + language, + voiceType, + greeting, + prompt, + coze_token, + coze_bot_id, + coze_base_url, + } = config const data = { request_id: genUUID(), channel_name: channel, @@ -42,11 +58,14 @@ export const apiStartService = async (config: StartRequestConfig): Promise graph_name: graphName, language, voice_type: voiceType, - greeting: greeting ? greeting : undefined, - prompt: prompt ? prompt : undefined + greeting: greeting ?? undefined, + prompt: prompt ?? undefined, + coze_token: coze_token ?? undefined, + coze_bot_id: coze_bot_id ?? undefined, + coze_base_url: coze_base_url ?? undefined, } let resp: any = await axios.post(url, data) - resp = (resp.data) || {} + resp = resp.data || {} return resp } @@ -55,10 +74,10 @@ export const apiStopService = async (channel: string) => { const url = `/api/agents/stop` const data = { request_id: genUUID(), - channel_name: channel + channel_name: channel, } let resp: any = await axios.post(url, data) - resp = (resp.data) || {} + resp = resp.data || {} return resp } @@ -66,14 +85,18 @@ export const apiGetDocumentList = async () => { // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL const url = `/api/vector/document/preset/list` let resp: any = await axios.get(url) - resp = (resp.data) || {} + resp = resp.data || {} if (resp.code !== "0") { throw new Error(resp.msg) } return resp } -export const apiUpdateDocument = async (options: { channel: string, collection: string, fileName: string }) => { +export const apiUpdateDocument = async (options: { + channel: string + collection: string + fileName: string +}) => { // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL const url = `/api/vector/document/update` const { channel, collection, fileName } = options @@ -81,23 +104,22 @@ export const apiUpdateDocument = async (options: { channel: string, collection: request_id: genUUID(), channel_name: channel, collection: collection, - file_name: fileName + file_name: fileName, } let resp: any = await axios.post(url, data) - resp = (resp.data) || {} + resp = resp.data || {} return resp } - -// ping/pong +// ping/pong export const apiPing = async (channel: string) => { // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL const url = `/api/agents/ping` const data = { request_id: genUUID(), - channel_name: channel + channel_name: channel, } let resp: any = await axios.post(url, data) - resp = (resp.data) || {} + resp = resp.data || {} return resp } diff --git a/demo/src/common/storage.ts b/demo/src/common/storage.ts index 29bc673c..74df4295 100644 --- a/demo/src/common/storage.ts +++ b/demo/src/common/storage.ts @@ -1,8 +1,23 @@ -import { IAgentSettings, IOptions } from "@/types" -import { OPTIONS_KEY, DEFAULT_OPTIONS, AGENT_SETTINGS_KEY, DEFAULT_AGENT_SETTINGS } from "./constant" +import { IAgentSettings, IOptions, ICozeSettings } from "@/types" +import { + OPTIONS_KEY, + DEFAULT_OPTIONS, + AGENT_SETTINGS_KEY, + DEFAULT_AGENT_SETTINGS, + COZE_SETTINGS_KEY, + DEFAULT_COZE_SETTINGS, +} from "./constant" -export const getOptionsFromLocal = (): {options:IOptions, settings: IAgentSettings} => { - let data = {options: DEFAULT_OPTIONS, settings: DEFAULT_AGENT_SETTINGS} +export const getOptionsFromLocal = (): { + options: IOptions + settings: IAgentSettings + cozeSettings: ICozeSettings +} => { + let data = { + options: DEFAULT_OPTIONS, + settings: DEFAULT_AGENT_SETTINGS, + cozeSettings: DEFAULT_COZE_SETTINGS, + } if (typeof window !== "undefined") { const options = localStorage.getItem(OPTIONS_KEY) if (options) { @@ -12,11 +27,14 @@ export const getOptionsFromLocal = (): {options:IOptions, settings: IAgentSettin if (settings) { data.settings = JSON.parse(settings) } + const cozeSettings = localStorage.getItem(COZE_SETTINGS_KEY) + if (cozeSettings) { + data.cozeSettings = JSON.parse(cozeSettings) + } } return data } - export const setOptionsToLocal = (options: IOptions) => { if (typeof window !== "undefined") { localStorage.setItem(OPTIONS_KEY, JSON.stringify(options)) @@ -28,3 +46,25 @@ export const setAgentSettingsToLocal = (settings: IAgentSettings) => { localStorage.setItem(AGENT_SETTINGS_KEY, JSON.stringify(settings)) } } + +export const setCozeSettingsToLocal = (settings: ICozeSettings) => { + if (typeof window !== "undefined") { + localStorage.setItem(COZE_SETTINGS_KEY, JSON.stringify(settings)) + } +} + +export const resetSettingsByKeys = (keys: string | string[]) => { + if (typeof window !== "undefined") { + if (Array.isArray(keys)) { + keys.forEach((key) => { + localStorage.removeItem(key) + }) + } else { + localStorage.removeItem(keys) + } + } +} + +export const resetCozeSettings = () => { + resetSettingsByKeys(COZE_SETTINGS_KEY) +} diff --git a/demo/src/components/Dialog/Settings.tsx b/demo/src/components/Dialog/Settings.tsx index 78c02249..247c8742 100644 --- a/demo/src/components/Dialog/Settings.tsx +++ b/demo/src/components/Dialog/Settings.tsx @@ -23,21 +23,137 @@ import { FormMessage, } from "@/components/ui/form" import { Textarea } from "@/components/ui/textarea" -import { SettingsIcon } from "lucide-react" +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" +import { + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectLabel, + SelectTrigger, + SelectValue, +} from "@/components/ui/select" +import { SettingsIcon, EraserIcon, ShieldCheckIcon } from "lucide-react" import { zodResolver } from "@hookform/resolvers/zod" import { useForm } from "react-hook-form" import { z } from "zod" +import { toast } from "sonner" +import { useAppDispatch, useAppSelector, ECozeBaseUrl } from "@/common" +import { + setAgentSettings, + setCozeSettings, + resetCozeSettings, + setGlobalSettingsDialog, +} from "@/store/reducers/global" + +const TABS_OPTIONS = [ + { + label: "Agent", + value: "agent", + }, + { + label: "Coze", + value: "coze", + }, +] + +export const useSettingsTabs = () => { + const [tabs, setTabs] = React.useState(TABS_OPTIONS) + + const graphName = useAppSelector((state) => state.global.graphName) + + const enableCozeSettingsMemo = React.useMemo(() => { + return isCozeGraph(graphName) + }, [graphName]) + + React.useEffect(() => { + if (enableCozeSettingsMemo) { + setTabs((prev) => + prev.find((tab) => tab.value === "coze") + ? prev + : [...prev, { label: "Coze", value: "coze" }], + ) + } else { + setTabs((prev) => prev.filter((tab) => tab.value !== "coze")) + } + }, [enableCozeSettingsMemo]) + + return tabs +} + +export default function SettingsDialog() { + const dispatch = useAppDispatch() + const globalSettingsDialog = useAppSelector( + (state) => state.global.globalSettingsDialog, + ) + + const tabs = useSettingsTabs() + + const handleClose = () => { + dispatch(setGlobalSettingsDialog({ open: false, tab: undefined })) + } -import { useAppDispatch, useAppSelector } from "@/common" -import { setAgentSettings } from "@/store/reducers/global" + return ( + + dispatch(setGlobalSettingsDialog({ open, tab: undefined })) + } + > + + + + + + Settings + + + {tabs.length > 1 && ( + + {tabs.map((tab) => ( + + {tab.label} + + ))} + + )} + + + + + + + + + + ) +} const formSchema = z.object({ - greeting: z.string(), - prompt: z.string(), + greeting: z.string().optional(), + prompt: z.string().optional(), }) -export default function SettingsDialog() { - const [open, setOpen] = React.useState(false) +export function CommonAgentSettingsTab(props: { + handleClose?: () => void + handleSubmit?: (values: z.infer) => void +}) { + const { handleSubmit } = props const dispatch = useAppDispatch() const agentSettings = useAppSelector((state) => state.global.agentSettings) @@ -53,64 +169,184 @@ export default function SettingsDialog() { function onSubmit(values: z.infer) { console.log("Form Values:", values) dispatch(setAgentSettings(values)) - setOpen(false) + handleSubmit?.(values) } return ( - - - - - - - Settings - +
+ + ( + + Greeting + +