From 5a87ac9d78c93ff644dbae8c2615d8e23e13e3f9 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 31 Oct 2024 01:45:19 +0000 Subject: [PATCH 1/6] feat: init default_async_llm_extension_python and default_async_llm_tool_extension_python --- .../README.md | 29 +++++ .../__init__.py | 6 + .../addon.py | 19 ++++ .../extension.py | 38 +++++++ .../manifest.json | 103 ++++++++++++++++++ .../property.json | 1 + .../requirements.txt | 0 .../README.md | 29 +++++ .../__init__.py | 6 + .../addon.py | 20 ++++ .../extension.py | 30 +++++ .../manifest.json | 67 ++++++++++++ .../property.json | 1 + .../requirements.txt | 0 14 files changed, 349 insertions(+) create mode 100644 agents/ten_packages/extension/default_async_llm_extension_python/README.md create mode 100644 agents/ten_packages/extension/default_async_llm_extension_python/__init__.py create mode 100644 agents/ten_packages/extension/default_async_llm_extension_python/addon.py create mode 100644 agents/ten_packages/extension/default_async_llm_extension_python/extension.py create mode 100644 agents/ten_packages/extension/default_async_llm_extension_python/manifest.json create mode 100644 agents/ten_packages/extension/default_async_llm_extension_python/property.json create mode 100644 agents/ten_packages/extension/default_async_llm_extension_python/requirements.txt create mode 100644 agents/ten_packages/extension/default_async_llm_tool_extension_python/README.md create mode 100644 agents/ten_packages/extension/default_async_llm_tool_extension_python/__init__.py create mode 100644 agents/ten_packages/extension/default_async_llm_tool_extension_python/addon.py create mode 100644 agents/ten_packages/extension/default_async_llm_tool_extension_python/extension.py create mode 100644 agents/ten_packages/extension/default_async_llm_tool_extension_python/manifest.json create mode 100644 agents/ten_packages/extension/default_async_llm_tool_extension_python/property.json create mode 100644 agents/ten_packages/extension/default_async_llm_tool_extension_python/requirements.txt diff --git a/agents/ten_packages/extension/default_async_llm_extension_python/README.md b/agents/ten_packages/extension/default_async_llm_extension_python/README.md new file mode 100644 index 00000000..31ee575e --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_extension_python/README.md @@ -0,0 +1,29 @@ +# default_async_llm_extension_python + + + +## Features + + + +- xxx feature + +## API + +Refer to `api` definition in [manifest.json] and default values in [property.json](property.json). + + + +## Development + +### Build + + + +### Unit test + + + +## Misc + + diff --git a/agents/ten_packages/extension/default_async_llm_extension_python/__init__.py b/agents/ten_packages/extension/default_async_llm_extension_python/__init__.py new file mode 100644 index 00000000..72593ab2 --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_extension_python/__init__.py @@ -0,0 +1,6 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from . import addon diff --git a/agents/ten_packages/extension/default_async_llm_extension_python/addon.py b/agents/ten_packages/extension/default_async_llm_extension_python/addon.py new file mode 100644 index 00000000..5d095832 --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_extension_python/addon.py @@ -0,0 +1,19 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import ( + Addon, + register_addon_as_extension, + TenEnv, +) +from .extension import DefaultAsyncLLMExtension + + +@register_addon_as_extension("default_async_llm_extension_python") +class DefaultAsyncLLMExtensionAddon(Addon): + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + ten_env.log_info("on_create_instance") + ten_env.on_create_instance_done( + DefaultAsyncLLMExtension(name), context) diff --git a/agents/ten_packages/extension/default_async_llm_extension_python/extension.py b/agents/ten_packages/extension/default_async_llm_extension_python/extension.py new file mode 100644 index 00000000..ed857403 --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_extension_python/extension.py @@ -0,0 +1,38 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import AsyncTenEnv +from ten_ai_base import ( + AsyncLLMBaseExtension, LLMCallCompletionArgs, LLMDataCompletionArgs, LLMToolMetadata +) + + +class DefaultAsyncLLMExtension(AsyncLLMBaseExtension): + async def on_start(self, ten_env: AsyncTenEnv) -> None: + await super().on_start(ten_env) + + """Implement this method to construct and start your resources.""" + ten_env.log_debug("TODO: on_start") + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + await super().on_stop(ten_env) + + """Implement this method to stop and destruct your resources.""" + ten_env.log_debug("TODO: on_stop") + + async def on_call_chat_completion(self, ten_env: AsyncTenEnv, **kargs: LLMCallCompletionArgs) -> None: + """Called when a chat completion is requested by cmd call. Implement this method to process the chat completion.""" + ten_env.log_debug("TODO: on_call_chat_completion") + + async def on_data_chat_completion(self, ten_env: AsyncTenEnv, **kargs: LLMDataCompletionArgs) -> None: + """ + Called when a chat completion is requested by data input. Implement this method to process the chat completion. + Note that this method is stream-based, and it should consider supporting local context caching. + """ + ten_env.log_debug("TODO: on_data_chat_completion") + + async def on_tools_update(self, ten_env: AsyncTenEnv, tool: LLMToolMetadata) -> None: + """Called when a new tool is registered. Implement this method to process the new tool.""" + ten_env.log_debug("TODO: on_tools_update") diff --git a/agents/ten_packages/extension/default_async_llm_extension_python/manifest.json b/agents/ten_packages/extension/default_async_llm_extension_python/manifest.json new file mode 100644 index 00000000..723f1f90 --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_extension_python/manifest.json @@ -0,0 +1,103 @@ +{ + "type": "extension", + "name": "default_async_llm_extension_python", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_ai_base", + "version": "0.1.0" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "requirements.txt", + "**.tent", + "**.py", + "README.md" + ] + }, + "api": { + "property": {}, + "cmd_in": [ + { + "name": "tool_register", + "property": { + "tool": { + "type": "string" + } + }, + "required": [ + "tool" + ] + }, + { + "name": "call_chat_completion", + "property": { + "messages": { + "type": "string" + }, + "stream": { + "type": "bool" + } + }, + "required": [ + "messages" + ], + "result": { + "property": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ] + } + }, + { + "name": "flush" + } + ], + "cmd_out": [ + { + "name": "flush" + } + ], + "data_in": [ + { + "name": "text_data", + "property": { + "text": { + "type": "string" + }, + "is_final": { + "type": "bool" + } + }, + "required": [ + "text" + ] + } + ], + "data_out": [ + { + "name": "text_data", + "property": { + "text": { + "type": "string" + }, + "end_of_segment": { + "type": "bool" + } + }, + "required": [ + "text", + "end_of_segment" + ] + } + ] + } +} \ No newline at end of file diff --git a/agents/ten_packages/extension/default_async_llm_extension_python/property.json b/agents/ten_packages/extension/default_async_llm_extension_python/property.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_extension_python/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/agents/ten_packages/extension/default_async_llm_extension_python/requirements.txt b/agents/ten_packages/extension/default_async_llm_extension_python/requirements.txt new file mode 100644 index 00000000..e69de29b diff --git a/agents/ten_packages/extension/default_async_llm_tool_extension_python/README.md b/agents/ten_packages/extension/default_async_llm_tool_extension_python/README.md new file mode 100644 index 00000000..8b26a3ab --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_tool_extension_python/README.md @@ -0,0 +1,29 @@ +# default_async_llm_tool_extension_python + + + +## Features + + + +- xxx feature + +## API + +Refer to `api` definition in [manifest.json] and default values in [property.json](property.json). + + + +## Development + +### Build + + + +### Unit test + + + +## Misc + + diff --git a/agents/ten_packages/extension/default_async_llm_tool_extension_python/__init__.py b/agents/ten_packages/extension/default_async_llm_tool_extension_python/__init__.py new file mode 100644 index 00000000..72593ab2 --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_tool_extension_python/__init__.py @@ -0,0 +1,6 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from . import addon diff --git a/agents/ten_packages/extension/default_async_llm_tool_extension_python/addon.py b/agents/ten_packages/extension/default_async_llm_tool_extension_python/addon.py new file mode 100644 index 00000000..8cab0dd2 --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_tool_extension_python/addon.py @@ -0,0 +1,20 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import ( + Addon, + register_addon_as_extension, + TenEnv, +) +from .extension import DefaultAsyncLLMToolExtension + + +@register_addon_as_extension("default_async_llm_tool_extension_python") +class DefaultAsyncLLMToolExtensionAddon(Addon): + + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + ten_env.log_info("on_create_instance") + ten_env.on_create_instance_done( + DefaultAsyncLLMToolExtension(name), context) diff --git a/agents/ten_packages/extension/default_async_llm_tool_extension_python/extension.py b/agents/ten_packages/extension/default_async_llm_tool_extension_python/extension.py new file mode 100644 index 00000000..fdc68271 --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_tool_extension_python/extension.py @@ -0,0 +1,30 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import ( + TenEnv, + AsyncTenEnv, +) +from ten_ai_base import AsyncLLMToolBaseExtension, LLMToolMetadata, LLMToolResult + + +class DefaultAsyncLLMToolExtension(AsyncLLMToolBaseExtension): + async def on_start(self, ten_env: AsyncTenEnv) -> None: + await super().on_start(ten_env) + + """Implement this method to construct and start your resources.""" + ten_env.log_debug("TODO: on_start") + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + await super().on_stop(ten_env) + + """Implement this method to stop and destruct your resources.""" + ten_env.log_debug("TODO: on_stop") + + def get_tool_metadata(self, ten_env: TenEnv) -> list[LLMToolMetadata]: + ten_env.log_debug("TODO: get_tool_metadata") + + async def run_tool(self, ten_env: AsyncTenEnv, name: str, args: dict) -> LLMToolResult: + ten_env.log_debug(f"TODO: run_tool {name} {args}") diff --git a/agents/ten_packages/extension/default_async_llm_tool_extension_python/manifest.json b/agents/ten_packages/extension/default_async_llm_tool_extension_python/manifest.json new file mode 100644 index 00000000..6cb22f0d --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_tool_extension_python/manifest.json @@ -0,0 +1,67 @@ +{ + "type": "extension", + "name": "default_async_llm_tool_extension_python", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_ai_base", + "version": "0.1.0" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "requirements.txt", + "**.tent", + "**.py", + "README.md" + ] + }, + "api": { + "property": {}, + "cmd_in": [ + { + "name": "tool_call", + "property": { + "name": { + "type": "string" + }, + "arguments": { + "type": "string" + } + }, + "required": [ + "name", + "arguments" + ], + "result": [ + { + "property": { + "tool_result": { + "type": "string" + } + }, + "required": [ + "tool_result" + ] + } + ] + } + ], + "cmd_out": [ + { + "name": "tool_register", + "property": { + "tool": { + "type": "string" + } + }, + "required": [ + "tool" + ] + } + ] + } +} \ No newline at end of file diff --git a/agents/ten_packages/extension/default_async_llm_tool_extension_python/property.json b/agents/ten_packages/extension/default_async_llm_tool_extension_python/property.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/agents/ten_packages/extension/default_async_llm_tool_extension_python/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/agents/ten_packages/extension/default_async_llm_tool_extension_python/requirements.txt b/agents/ten_packages/extension/default_async_llm_tool_extension_python/requirements.txt new file mode 100644 index 00000000..e69de29b From d48e3f5b2849899a90e0f4ca7ff10147e01641fd Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 31 Oct 2024 01:46:32 +0000 Subject: [PATCH 2/6] feat: manifest for ten_ai_base package --- .../interface/ten_ai_base/__init__.py | 15 ++++++++++++++ .../system/ten_ai_base/manifest.json | 20 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 agents/ten_packages/system/ten_ai_base/manifest.json diff --git a/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/__init__.py b/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/__init__.py index da402faf..5f40c599 100644 --- a/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/__init__.py +++ b/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/__init__.py @@ -3,3 +3,18 @@ # Licensed under the Apache License, Version 2.0. # See the LICENSE file for more information. # + +from .types import LLMCallCompletionArgs, LLMDataCompletionArgs, LLMToolMetadata, LLMToolResult +from .llm import AsyncLLMBaseExtension +from .llm_tool import AsyncLLMToolBaseExtension + +# Specify what should be imported when a user imports * from the +# ten_ai_base package. +__all__ = [ + "LLMToolMetadata", + "LLMToolResult", + "LLMCallCompletionArgs", + "LLMDataCompletionArgs", + "AsyncLLMBaseExtension", + "AsyncLLMToolBaseExtension" +] diff --git a/agents/ten_packages/system/ten_ai_base/manifest.json b/agents/ten_packages/system/ten_ai_base/manifest.json new file mode 100644 index 00000000..a67d065d --- /dev/null +++ b/agents/ten_packages/system/ten_ai_base/manifest.json @@ -0,0 +1,20 @@ +{ + "type": "system", + "name": "ten_ai_base", + "version": "0.1.0", + "package": { + "include": [ + "manifest.json", + "interface/**", + "requirements.txt" + ] + }, + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.3" + } + ], + "api": {} +} \ No newline at end of file From 43ff8ce4f22a4cb977aa84aec1988a78ee6d852b Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 31 Oct 2024 01:50:22 +0000 Subject: [PATCH 3/6] fix: ten env --- .../ten_ai_base/interface/ten_ai_base/llm.py | 41 +++++++++---------- .../interface/ten_ai_base/llm_tool.py | 38 ++++++++--------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm.py b/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm.py index bc2665af..8e2c659d 100644 --- a/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm.py +++ b/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm.py @@ -5,13 +5,9 @@ # from abc import ABC, abstractmethod import asyncio -from enum import Enum -from typing import Optional, TypedDict, Union -from pydantic import BaseModel from ten import ( AsyncExtension, - TenEnv, Data, ) from ten.async_ten_env import AsyncTenEnv @@ -22,6 +18,7 @@ from .helper import AsyncQueue import json + class AsyncLLMBaseExtension(AsyncExtension, ABC): """ Base class for implementing a Language Model Extension. @@ -32,6 +29,7 @@ class AsyncLLMBaseExtension(AsyncExtension, ABC): Override on_call_chat_completion and on_data_chat_completion to implement the chat completion logic. """ # Create the queue for message processing + def __init__(self, name: str): super().__init__(name) self.queue = AsyncQueue() @@ -40,19 +38,19 @@ def __init__(self, name: str): self.current_task = None self.hit_default_cmd = False - async def on_init(self, ten_env: TenEnv) -> None: + async def on_init(self, ten_env: AsyncTenEnv) -> None: ten_env.log_debug("on_init") - async def on_start(self, ten_env: TenEnv) -> None: + async def on_start(self, ten_env: AsyncTenEnv) -> None: ten_env.log_debug("on_start") self.loop = asyncio.get_event_loop() self.loop.create_task(self._process_queue(ten_env)) - async def on_stop(self, ten_env: TenEnv) -> None: + async def on_stop(self, ten_env: AsyncTenEnv) -> None: ten_env.log_debug("on_stop") - async def on_deinit(self, ten_env: TenEnv) -> None: + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: ten_env.log_debug("on_deinit") async def on_cmd(self, async_ten_env: AsyncTenEnv, cmd: Cmd) -> None: @@ -64,23 +62,26 @@ async def on_cmd(self, async_ten_env: AsyncTenEnv, cmd: Cmd) -> None: async_ten_env.log_debug(f"on_cmd name {cmd_name}") if cmd_name == CMD_TOOL_REGISTER: try: - tool_metadata_json = json.loads(cmd.get_property_to_json(CMD_PROPERTY_TOOL)) + tool_metadata_json = json.loads( + cmd.get_property_to_json(CMD_PROPERTY_TOOL)) async_ten_env.log_info(f"register tool: {tool_metadata_json}") - tool_metadata = LLMToolMetadata.model_validate_json(tool_metadata_json) + tool_metadata = LLMToolMetadata.model_validate_json( + tool_metadata_json) async with self.available_tools_lock: self.available_tools.append(tool_metadata) await self.on_tools_update(async_ten_env, tool_metadata) - async_ten_env.return_result(CmdResult.create(StatusCode.OK), cmd) + async_ten_env.return_result( + CmdResult.create(StatusCode.OK), cmd) except Exception as err: async_ten_env.log_warn(f"on_cmd failed: {err}") - async_ten_env.return_result(CmdResult.create(StatusCode.ERROR), cmd) - + async_ten_env.return_result( + CmdResult.create(StatusCode.ERROR), cmd) async def queue_input_item(self, prepend: bool = False, **kargs: LLMDataCompletionArgs): """Queues an input item for processing.""" await self.queue.put(kargs, prepend) - async def flush_input_items(self, ten_env: TenEnv): + async def flush_input_items(self, ten_env: AsyncTenEnv): """Flushes the self.queue and cancels the current task.""" # Flush the queue using the new flush method await self.queue.flush() @@ -90,7 +91,7 @@ async def flush_input_items(self, ten_env: TenEnv): ten_env.log_info("Cancelling the current task during flush.") self.current_task.cancel() - def send_text_output(self, ten_env: TenEnv, sentence: str, end_of_segment: bool): + def send_text_output(self, ten_env: AsyncTenEnv, sentence: str, end_of_segment: bool): try: output_data = Data.create(DATA_OUTPUT_NAME) output_data.set_property_string( @@ -108,12 +109,12 @@ def send_text_output(self, ten_env: TenEnv, sentence: str, end_of_segment: bool) ) @abstractmethod - async def on_call_chat_completion(self, ten_env: TenEnv, **kargs: LLMCallCompletionArgs) -> None: + async def on_call_chat_completion(self, ten_env: AsyncTenEnv, **kargs: LLMCallCompletionArgs) -> None: """Called when a chat completion is requested by cmd call. Implement this method to process the chat completion.""" pass @abstractmethod - async def on_data_chat_completion(self, ten_env: TenEnv, **kargs: LLMDataCompletionArgs) -> None: + async def on_data_chat_completion(self, ten_env: AsyncTenEnv, **kargs: LLMDataCompletionArgs) -> None: """ Called when a chat completion is requested by data input. Implement this method to process the chat completion. Note that this method is stream-based, and it should consider supporting local context caching. @@ -121,11 +122,11 @@ async def on_data_chat_completion(self, ten_env: TenEnv, **kargs: LLMDataComplet pass @abstractmethod - async def on_tools_update(self, ten_env: TenEnv, tool: LLMToolMetadata) -> None: + async def on_tools_update(self, ten_env: AsyncTenEnv, tool: LLMToolMetadata) -> None: """Called when a new tool is registered. Implement this method to process the new tool.""" pass - async def _process_queue(self, ten_env: TenEnv): + async def _process_queue(self, ten_env: AsyncTenEnv): """Asynchronously process queue items one by one.""" while True: # Wait for an item to be available in the queue @@ -137,5 +138,3 @@ async def _process_queue(self, ten_env: TenEnv): await self.current_task # Wait for the current task to finish or be cancelled except asyncio.CancelledError: ten_env.log_info(f"Task cancelled: {args}") - - \ No newline at end of file diff --git a/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm_tool.py b/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm_tool.py index bf4c76ff..de2a3052 100644 --- a/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm_tool.py +++ b/agents/ten_packages/system/ten_ai_base/interface/ten_ai_base/llm_tool.py @@ -1,11 +1,8 @@ from abc import ABC, abstractmethod -import asyncio -from enum import Enum -from typing import Optional, TypedDict from ten import ( AsyncExtension, - TenEnv, Data, + TenEnv, ) from ten.async_ten_env import AsyncTenEnv from ten.audio_frame import AudioFrame @@ -16,25 +13,26 @@ from .const import CMD_TOOL_REGISTER, CMD_TOOL_CALL, CMD_PROPERTY_TOOL, CMD_PROPERTY_RESULT import json + class AsyncLLMToolBaseExtension(AsyncExtension, ABC): async def on_start(self, ten_env: AsyncTenEnv) -> None: ten_env.log_debug("on_start") tools = self.get_tool_metadata() for tool in tools: ten_env.log_info(f"tool: {tool}") - c:Cmd = Cmd.create(CMD_TOOL_REGISTER) - c.set_property_from_json(CMD_PROPERTY_TOOL, json.dumps(tool.model_dump_json())) + c: Cmd = Cmd.create(CMD_TOOL_REGISTER) + c.set_property_from_json( + CMD_PROPERTY_TOOL, json.dumps(tool.model_dump_json())) await ten_env.send_cmd(c) ten_env.log_info(f"tool registered, {tool}") - - async def on_stop(self, ten_env: TenEnv) -> None: + async def on_stop(self, ten_env: AsyncTenEnv) -> None: ten_env.log_debug("on_stop") - async def on_deinit(self, ten_env: TenEnv) -> None: + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: ten_env.log_debug("on_deinit") - async def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: cmd_name = cmd.get_name() ten_env.log_debug("on_cmd name {}".format(cmd_name)) @@ -42,31 +40,33 @@ async def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: try: tool_name = cmd.get_property_string("name") tool_args = json.loads(cmd.get_property_to_json("arguments")) - ten_env.log_debug(f"tool_name: {tool_name}, tool_args: {tool_args}") + ten_env.log_debug( + f"tool_name: {tool_name}, tool_args: {tool_args}") result = await self.run_tool(tool_name, tool_args) - cmd_result:CmdResult = CmdResult.create(StatusCode.OK) - cmd_result.set_property_from_json(CMD_PROPERTY_RESULT, json.dumps(result.model_dump_json())) + cmd_result: CmdResult = CmdResult.create(StatusCode.OK) + cmd_result.set_property_from_json( + CMD_PROPERTY_RESULT, json.dumps(result.model_dump_json())) ten_env.return_result(cmd_result, cmd) ten_env.log_debug(f"tool result done, {result}") except Exception as err: ten_env.log_warn(f"on_cmd failed: {err}") ten_env.return_result(CmdResult.create(StatusCode.ERROR), cmd) - async def on_data(self, ten_env: TenEnv, data: Data) -> None: + async def on_data(self, ten_env: AsyncTenEnv, data: Data) -> None: data_name = data.get_name() ten_env.log_debug("on_data name {}".format(data_name)) # TODO: process data pass - async def on_audio_frame(self, ten_env: TenEnv, audio_frame: AudioFrame) -> None: + async def on_audio_frame(self, ten_env: AsyncTenEnv, audio_frame: AudioFrame) -> None: audio_frame_name = audio_frame.get_name() ten_env.log_debug("on_audio_frame name {}".format(audio_frame_name)) # TODO: process audio frame pass - async def on_video_frame(self, ten_env: TenEnv, video_frame: VideoFrame) -> None: + async def on_video_frame(self, ten_env: AsyncTenEnv, video_frame: VideoFrame) -> None: video_frame_name = video_frame.get_name() ten_env.log_debug("on_video_frame name {}".format(video_frame_name)) @@ -74,9 +74,9 @@ async def on_video_frame(self, ten_env: TenEnv, video_frame: VideoFrame) -> None pass @abstractmethod - def get_tool_metadata(self) -> list[LLMToolMetadata]: + def get_tool_metadata(self, ten_env: TenEnv) -> list[LLMToolMetadata]: pass @abstractmethod - async def run_tool(self, name: str, args: dict) -> LLMToolResult: - pass \ No newline at end of file + async def run_tool(self, ten_env: AsyncTenEnv, name: str, args: dict) -> LLMToolResult: + pass From 32a60224ba3b44153eeb59234a413994fa85c223 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 31 Oct 2024 01:51:13 +0000 Subject: [PATCH 4/6] chore: ignore installed sdk --- agents/.gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/agents/.gitignore b/agents/.gitignore index 7e8da43e..05d39a92 100644 --- a/agents/.gitignore +++ b/agents/.gitignore @@ -4,7 +4,10 @@ ten_packages/extension/agora_rtc ten_packages/extension/azure_tts ten_packages/extension/agora_sess_ctrl ten_packages/extension/py_init_extension_cpp -ten_packages/system +ten_packages/system/agora_rtc_sdk +ten_packages/system/azure_speech_sdk +ten_packages/system/nlohmann_json +ten_packages/system/ten_runtime* .ten agoradns.dat agorareport.dat From c10a9fb66d9cf20e59218f4c4ebad6497a592766 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 31 Oct 2024 01:59:02 +0000 Subject: [PATCH 5/6] fix: syntax --- .../manifest.json | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/agents/ten_packages/extension/default_async_llm_tool_extension_python/manifest.json b/agents/ten_packages/extension/default_async_llm_tool_extension_python/manifest.json index 6cb22f0d..b6b44832 100644 --- a/agents/ten_packages/extension/default_async_llm_tool_extension_python/manifest.json +++ b/agents/ten_packages/extension/default_async_llm_tool_extension_python/manifest.json @@ -36,18 +36,16 @@ "name", "arguments" ], - "result": [ - { - "property": { - "tool_result": { - "type": "string" - } - }, - "required": [ - "tool_result" - ] - } - ] + "result": { + "property": { + "tool_result": { + "type": "string" + } + }, + "required": [ + "tool_result" + ] + } } ], "cmd_out": [ From 417a6a8771de2481ac02dbeab675f11b9b8cd8d2 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 31 Oct 2024 01:59:23 +0000 Subject: [PATCH 6/6] fix: clean --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7163e3c7..229d4813 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ clean: clean-agents clean-agents: @echo ">> clean agents" - rm -rf agents/bin/worker agents/out agents/interface agents/include agents/lib agents/lib64 agents/ten_packages/system agents/ten_packages/extension_group agents/.release + rm -rf agents/bin/worker agents/out agents/interface agents/include agents/lib agents/lib64 agents/ten_packages/system/ten_runtime* agents/ten_packages/system/agora_rtc_sdk agents/ten_packages/system/azure_speech_sdk agents/ten_packages/system/nlohmann_json agents/.release @echo ">> done" docker-build-playground: