From cffa1170488d956d288ad2d8b3beb465c3761e67 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Tue, 6 Aug 2024 20:31:03 -0700 Subject: [PATCH 01/34] feat: add http_server_extension --- .../extension/http_server_python/__init__.py | 4 + .../http_server_python/http_server_addon.py | 14 +++ .../http_server_extension.py | 89 +++++++++++++++++++ .../addon/extension/http_server_python/log.py | 13 +++ .../http_server_python/manifest.json | 30 +++++++ .../http_server_python/property.json | 4 + 6 files changed, 154 insertions(+) create mode 100644 agents/addon/extension/http_server_python/__init__.py create mode 100644 agents/addon/extension/http_server_python/http_server_addon.py create mode 100644 agents/addon/extension/http_server_python/http_server_extension.py create mode 100644 agents/addon/extension/http_server_python/log.py create mode 100644 agents/addon/extension/http_server_python/manifest.json create mode 100644 agents/addon/extension/http_server_python/property.json diff --git a/agents/addon/extension/http_server_python/__init__.py b/agents/addon/extension/http_server_python/__init__.py new file mode 100644 index 00000000..0bcc331c --- /dev/null +++ b/agents/addon/extension/http_server_python/__init__.py @@ -0,0 +1,4 @@ +from . import http_server_addon +from .log import logger + +logger.info("http_server_python extension loaded") diff --git a/agents/addon/extension/http_server_python/http_server_addon.py b/agents/addon/extension/http_server_python/http_server_addon.py new file mode 100644 index 00000000..b1884e81 --- /dev/null +++ b/agents/addon/extension/http_server_python/http_server_addon.py @@ -0,0 +1,14 @@ +from rte import ( + Addon, + register_addon_as_extension, + RteEnv, +) +from .log import logger +from .http_server_extension import HTTPServerExtension + + +@register_addon_as_extension("http_server_python") +class HTTPServerExtensionAddon(Addon): + def on_create_instance(self, rte: RteEnv, addon_name: str, context): + logger.info("on_create_instance") + rte.on_create_instance_done(HTTPServerExtension(addon_name), context) diff --git a/agents/addon/extension/http_server_python/http_server_extension.py b/agents/addon/extension/http_server_python/http_server_extension.py new file mode 100644 index 00000000..1092231e --- /dev/null +++ b/agents/addon/extension/http_server_python/http_server_extension.py @@ -0,0 +1,89 @@ +from rte import ( + Extension, + RteEnv, + Cmd, + StatusCode, + CmdResult, +) +from .log import logger +from http.server import HTTPServer, BaseHTTPRequestHandler +import threading +from functools import partial + + +class HTTPHandler(BaseHTTPRequestHandler): + def __init__(self, rte, *args, directory=None, **kwargs): + logger.info("new handler: %s %s %s", directory, args, kwargs) + self.rte = rte + super().__init__(*args, **kwargs) + + def do_POST(self): + logger.info("post request incoming %s", self.path) + if self.path == "/cmd": + try: + content_length = int(self.headers["Content-Length"]) + input = self.rfile.read(content_length).decode("utf-8") + logger.info("incoming request %s", input) + self.rte.send_cmd( + Cmd.create_from_json(input), + lambda rte, result: logger.info( + "finish send_cmd from http server %s %s", input, result + ), + ) + self.send_response_only(200) + self.end_headers() + except Exception as e: + logger.warning("failed to handle request, err {}".format(e)) + self.send_response_only(500) + self.end_headers() + else: + logger.warning("invalid path: %s", self.path) + self.send_response_only(404) + self.end_headers() + + +class HTTPServerExtension(Extension): + def __init__(self, name: str): + super().__init__(name) + self.listen_addr = "127.0.0.1" + self.listen_port = 8888 + self.cmd_white_list = None + self.server = None + self.thread = None + + def on_start(self, rte: RteEnv): + self.listen_addr = rte.get_property_string("listen_addr") + self.listen_port = rte.get_property_int("listen_port") + """ + white_list = rte.get_property_string("cmd_white_list") + if len(white_list) > 0: + self.cmd_white_list = white_list.split(",") + """ + + logger.info( + "HTTPServerExtension on_start %s:%d, %s", + self.listen_addr, + self.listen_port, + self.cmd_white_list, + ) + + self.server = HTTPServer( + (self.listen_addr, self.listen_port), partial(HTTPHandler, rte) + ) + self.thread = threading.Thread(target=self.server.serve_forever) + self.thread.start() + + rte.on_start_done() + + def on_stop(self, rte: RteEnv): + logger.info("on_stop") + self.server.shutdown() + self.thread.join() + rte.on_stop_done() + + def on_cmd(self, rte: RteEnv, cmd: Cmd): + cmd_json = cmd.to_json() + logger.info("on_cmd json: " + cmd_json) + cmd_result = CmdResult.create(StatusCode.OK) + cmd_result.set_property_string("detail", "ok") + rte.return_result(cmd_result, cmd) diff --git a/agents/addon/extension/http_server_python/log.py b/agents/addon/extension/http_server_python/log.py new file mode 100644 index 00000000..a7b47d79 --- /dev/null +++ b/agents/addon/extension/http_server_python/log.py @@ -0,0 +1,13 @@ +import logging + +logger = logging.getLogger("http_server_python") +logger.setLevel(logging.INFO) + +formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - [%(filename)s:%(lineno)d] - %(message)s" +) + +console_handler = logging.StreamHandler() +console_handler.setFormatter(formatter) + +logger.addHandler(console_handler) diff --git a/agents/addon/extension/http_server_python/manifest.json b/agents/addon/extension/http_server_python/manifest.json new file mode 100644 index 00000000..bfc16037 --- /dev/null +++ b/agents/addon/extension/http_server_python/manifest.json @@ -0,0 +1,30 @@ +{ + "type": "extension", + "name": "http_server_python", + "version": "0.4.0", + "language": "python", + "dependencies": [ + { + "type": "system", + "name": "rte_runtime_python", + "version": "0.4" + } + ], + "publish": { + "include": [ + "manifest.json", + "property.json", + "**.py" + ] + }, + "api": { + "property": { + "listen_addr": { + "type": "string" + }, + "listen_port": { + "type": "int32" + } + } + } +} \ No newline at end of file diff --git a/agents/addon/extension/http_server_python/property.json b/agents/addon/extension/http_server_python/property.json new file mode 100644 index 00000000..27ae7c57 --- /dev/null +++ b/agents/addon/extension/http_server_python/property.json @@ -0,0 +1,4 @@ +{ + "listen_addr": "0.0.0.0", + "listen_port": 8888 +} \ No newline at end of file From 620679e90dc6d1f8a588cbfd7073053753246cd2 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Tue, 6 Aug 2024 23:49:11 -0700 Subject: [PATCH 02/34] feat: add aliyun_text_embedding --- .../aliyun_text_embedding/__init__.py | 4 + .../aliyun_text_embedding/embedding_addon.py | 14 ++ .../embedding_extension.py | 174 ++++++++++++++++++ .../extension/aliyun_text_embedding/log.py | 13 ++ .../aliyun_text_embedding/manifest.json | 94 ++++++++++ .../aliyun_text_embedding/property.json | 1 + .../aliyun_text_embedding/requirements.txt | 1 + 7 files changed, 301 insertions(+) create mode 100644 agents/addon/extension/aliyun_text_embedding/__init__.py create mode 100644 agents/addon/extension/aliyun_text_embedding/embedding_addon.py create mode 100644 agents/addon/extension/aliyun_text_embedding/embedding_extension.py create mode 100644 agents/addon/extension/aliyun_text_embedding/log.py create mode 100644 agents/addon/extension/aliyun_text_embedding/manifest.json create mode 100644 agents/addon/extension/aliyun_text_embedding/property.json create mode 100644 agents/addon/extension/aliyun_text_embedding/requirements.txt diff --git a/agents/addon/extension/aliyun_text_embedding/__init__.py b/agents/addon/extension/aliyun_text_embedding/__init__.py new file mode 100644 index 00000000..b06c946a --- /dev/null +++ b/agents/addon/extension/aliyun_text_embedding/__init__.py @@ -0,0 +1,4 @@ +from . import embedding_addon +from .log import logger + +logger.info("aliyun_text_embedding extension loaded") diff --git a/agents/addon/extension/aliyun_text_embedding/embedding_addon.py b/agents/addon/extension/aliyun_text_embedding/embedding_addon.py new file mode 100644 index 00000000..e149bbf6 --- /dev/null +++ b/agents/addon/extension/aliyun_text_embedding/embedding_addon.py @@ -0,0 +1,14 @@ +from rte import ( + Addon, + register_addon_as_extension, + RteEnv, +) +from .log import logger +from .embedding_extension import EmbeddingExtension + + +@register_addon_as_extension("aliyun_text_embedding") +class EmbeddingExtensionAddon(Addon): + def on_create_instance(self, rte: RteEnv, addon_name: str, context) -> None: + logger.info("on_create_instance") + rte.on_create_instance_done(EmbeddingExtension(addon_name), context) diff --git a/agents/addon/extension/aliyun_text_embedding/embedding_extension.py b/agents/addon/extension/aliyun_text_embedding/embedding_extension.py new file mode 100644 index 00000000..08077618 --- /dev/null +++ b/agents/addon/extension/aliyun_text_embedding/embedding_extension.py @@ -0,0 +1,174 @@ +from rte import ( + Extension, + RteEnv, + Cmd, + StatusCode, + CmdResult, +) + +import dashscope +import json +from typing import Generator, List +from http import HTTPStatus +from .log import logger +import threading, queue +from datetime import datetime + +CMD_EMBED = "embed" +CMD_EMBED_BATCH = "embed_batch" + +FIELD_KEY_EMBEDDING = "embedding" +FIELD_KEY_EMBEDDINGS = "embeddings" +FIELD_KEY_MESSAGE = "message" +FIELD_KEY_CODE = "code" + +DASHSCOPE_MAX_BATCH_SIZE = 6 + + +class EmbeddingExtension(Extension): + def __init__(self, name: str): + super().__init__(name) + self.api_key = "" + self.model = "" + + self.stop = False + self.queue = queue.Queue() + self.threads = [] + + # TODO: workaround to speech up the embedding process, + # should be replace by https://help.aliyun.com/zh/model-studio/developer-reference/text-embedding-batch-api?spm=a2c4g.11186623.0.0.24cb7453KSjdhC + # once v3 models supported + self.parallel = 10 + + def on_start(self, rte: RteEnv) -> None: + logger.info("on_start") + self.api_key = self.get_property_string(rte, "api_key", self.api_key) + self.model = self.get_property_string(rte, "model", self.api_key) + + dashscope.api_key = self.api_key + + for i in range(self.parallel): + thread = threading.Thread(target=self.async_handler, args=[i, rte]) + thread.start() + self.threads.append(thread) + + rte.on_start_done() + + def async_handler(self, index: int, rte: RteEnv): + logger.info("async_handler {} started".format(index)) + + while not self.stop: + cmd = self.queue.get() + if cmd is None: + break + + cmd_name = cmd.get_name() + logger.info( + "async_handler {} processing cmd {}".format(index, cmd_name)) + + if cmd_name == CMD_EMBED: + cmd_result = self.call_with_str(cmd.get_property_string("input")) + rte.return_result(cmd_result, cmd) + elif cmd_name == CMD_EMBED_BATCH: + list = json.loads(cmd.get_property_to_json("inputs")) + cmd_result = self.call_with_strs(list) + rte.return_result(cmd_result, cmd) + else: + logger.warning("unknown cmd {}".format(cmd_name)) + + logger.info("async_handler {} stopped".format(index)) + + def call_with_str(self, message: str) -> CmdResult: + start_time = datetime.now() + response = dashscope.TextEmbedding.call(model=self.model, input=message) + logger.info("embedding call finished for input [{}], status_code {}, cost {}ms".format(message, response.status_code, (datetime.now() - start_time).microseconds / 1000)) + + if response.status_code == HTTPStatus.OK: + cmd_result = CmdResult.create(StatusCode.OK) + cmd_result.set_property_from_json(FIELD_KEY_EMBEDDING, response.output["embeddings"][0]["embedding"]) + return cmd_result + else: + cmd_result = CmdResult.create(StatusCode.ERROR) + cmd_result.set_property_string(FIELD_KEY_CODE, response.status_code) + cmd_result.set_property_string(FIELD_KEY_MESSAGE, response.message) + return cmd_result + + def batched( + self, inputs: List, batch_size: int = DASHSCOPE_MAX_BATCH_SIZE + ) -> Generator[List, None, None]: + for i in range(0, len(inputs), batch_size): + yield inputs[i : i + batch_size] + + def call_with_strs(self, messages: List[str]) -> CmdResult: + start_time = datetime.now() + result = None # merge the results. + batch_counter = 0 + for batch in self.batched(messages): + response = dashscope.TextEmbedding.call(model=self.model, input=batch) + # logger.info("%s Received %s", batch, response) + if response.status_code == HTTPStatus.OK: + if result is None: + result = response.output + else: + for emb in response.output["embeddings"]: + emb["text_index"] += batch_counter + result["embeddings"].append(emb) + else: + logger.error("call %s failed, errmsg: %s", batch, response) + batch_counter += len(batch) + + logger.info("embedding call finished for inputs len {}, batch_counter {}, results len {}, cost {}ms ".format(len(messages), batch_counter, len(result["embeddings"]), (datetime.now() - start_time).microseconds / 1000)) + if result is not None: + cmd_result = CmdResult.create(StatusCode.OK) + cmd_result.set_property_from_json(FIELD_KEY_EMBEDDINGS, result["embeddings"]) + return cmd_result + else: + cmd_result = CmdResult.create(StatusCode.ERROR) + cmd_result.set_property_string(FIELD_KEY_MESSAGE, "All batch failed") + return cmd_result + + def on_stop(self, rte: RteEnv) -> None: + logger.info("on_stop") + self.stop = True + # clear queue + while not self.queue.empty(): + self.queue.get() + # put enough None to stop all threads + for thread in self.threads: + self.queue.put(None) + for thread in self.threads: + thread.join() + self.threads = [] + + rte.on_stop_done() + + def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: + cmd_name = cmd.get_name() + + if cmd_name in [CMD_EMBED, CMD_EMBED_BATCH]: + """ + // embed + { + "name": "embed", + "input": "hello" + } + + // embed_batch + { + "name": "embed_batch", + "inputs": ["hello", ...] + } + """ + + self.queue.put(cmd) + else: + logger.warning("unknown cmd {}".format(cmd_name)) + cmd_result = CmdResult.create(StatusCode.ERROR) + rte.return_result(cmd_result, cmd) + + def get_property_string(self, rte: RteEnv, key, default): + try: + return rte.get_property_string(key) + except Exception as e: + logger.warning(f"err: {e}") + return default diff --git a/agents/addon/extension/aliyun_text_embedding/log.py b/agents/addon/extension/aliyun_text_embedding/log.py new file mode 100644 index 00000000..2e90975a --- /dev/null +++ b/agents/addon/extension/aliyun_text_embedding/log.py @@ -0,0 +1,13 @@ +import logging + +logger = logging.getLogger("aliyun_text_embedding") +logger.setLevel(logging.INFO) + +formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - [%(filename)s:%(lineno)d] - %(message)s" +) + +console_handler = logging.StreamHandler() +console_handler.setFormatter(formatter) + +logger.addHandler(console_handler) diff --git a/agents/addon/extension/aliyun_text_embedding/manifest.json b/agents/addon/extension/aliyun_text_embedding/manifest.json new file mode 100644 index 00000000..ff9c46a2 --- /dev/null +++ b/agents/addon/extension/aliyun_text_embedding/manifest.json @@ -0,0 +1,94 @@ +{ + "type": "extension", + "name": "aliyun_text_embedding", + "version": "0.4.0", + "language": "python", + "dependencies": [ + { + "type": "system", + "name": "rte_runtime_python", + "version": "0.4" + } + ], + "api": { + "property": { + "api_key": { + "type": "string" + }, + "model": { + "type": "string" + } + }, + "cmd_in": [ + { + "name": "embed", + "property": { + "input": { + "type": "string" + } + }, + "required": [ + "input" + ], + "result": { + "property": { + "embedding": { + "type": "array", + "items": { + "type": "float64" + } + }, + "code": { + "type": "string" + }, + "message": { + "type": "string" + } + } + } + }, + { + "name": "embed_batch", + "property": { + "inputs": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "inputs" + ], + "result": { + "property": { + "embeddings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "embedding": { + "type": "array", + "items": { + "type": "float64" + } + }, + "text_index": { + "type": "int32" + } + } + } + }, + "code": { + "type": "string" + }, + "message": { + "type": "string" + } + } + } + } + ], + "cmd_out": [] + } +} \ No newline at end of file diff --git a/agents/addon/extension/aliyun_text_embedding/property.json b/agents/addon/extension/aliyun_text_embedding/property.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/agents/addon/extension/aliyun_text_embedding/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/agents/addon/extension/aliyun_text_embedding/requirements.txt b/agents/addon/extension/aliyun_text_embedding/requirements.txt new file mode 100644 index 00000000..5899464f --- /dev/null +++ b/agents/addon/extension/aliyun_text_embedding/requirements.txt @@ -0,0 +1 @@ +dashscope \ No newline at end of file From 4a3f27564a2a8fd9709d76cb6e4e928d20a33797 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Wed, 7 Aug 2024 00:10:46 -0700 Subject: [PATCH 03/34] feat: add aliyun_analyticdb_vector_storage --- .../__init__.py | 4 + .../client.py | 98 ++++ .../aliyun_analyticdb_vector_storage/log.py | 13 + .../manifest.json | 99 ++++ .../aliyun_analyticdb_vector_storage/model.py | 547 ++++++++++++++++++ .../property.json | 1 + .../requirements.txt | 1 + .../vector_storage_addon.py | 14 + .../vector_storage_extension.py | 215 +++++++ 9 files changed, 992 insertions(+) create mode 100644 agents/addon/extension/aliyun_analyticdb_vector_storage/__init__.py create mode 100644 agents/addon/extension/aliyun_analyticdb_vector_storage/client.py create mode 100644 agents/addon/extension/aliyun_analyticdb_vector_storage/log.py create mode 100644 agents/addon/extension/aliyun_analyticdb_vector_storage/manifest.json create mode 100644 agents/addon/extension/aliyun_analyticdb_vector_storage/model.py create mode 100644 agents/addon/extension/aliyun_analyticdb_vector_storage/property.json create mode 100644 agents/addon/extension/aliyun_analyticdb_vector_storage/requirements.txt create mode 100644 agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_addon.py create mode 100644 agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/__init__.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/__init__.py new file mode 100644 index 00000000..1af08bf4 --- /dev/null +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/__init__.py @@ -0,0 +1,4 @@ +from . import vector_storage_addon +from .log import logger + +logger.info("aliyun_analyticdb_vector_storage extension loaded") diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/client.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/client.py new file mode 100644 index 00000000..43c4f541 --- /dev/null +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/client.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +try: + from .log import logger +except ImportError: + from log import logger +import asyncio +import threading +from typing import Coroutine +from concurrent.futures import Future + + +from alibabacloud_gpdb20160503.client import Client as gpdb20160503Client +from alibabacloud_tea_openapi import models as open_api_models + + +# maybe need multiple clients +class AliGPDBClient: + def __init__(self, access_key_id, access_key_secret, endpoint): + self.stopEvent = asyncio.Event() + self.loop = None + self.tasks = asyncio.Queue() + self.access_key_id = access_key_id + self.access_key_secret = access_key_secret + self.endpoint = endpoint + self.client = self.create_client() + self.thread = threading.Thread( + target=asyncio.run, args=(self.__thread_routine(),) + ) + self.thread.start() + + async def stop_thread(self): + self.stopEvent.set() + + def create_client(self) -> gpdb20160503Client: + config = open_api_models.Config( + access_key_id=self.access_key_id, + access_key_secret=self.access_key_secret, + endpoint=self.endpoint, + ) + return gpdb20160503Client(config) + + def get(self) -> gpdb20160503Client: + return self.client + + def close(self): + if (self.loop is not None) and self.thread.is_alive(): + self.stopEvent.set() + asyncio.run_coroutine_threadsafe(self.stop_thread(), self.loop) + self.thread.join() + + async def __thread_routine(self): + logger.info("client __thread_routine start") + self.loop = asyncio.get_running_loop() + tasks = set() + while not self.stopEvent.is_set(): + if not self.tasks.empty(): + coro, future = await self.tasks.get() + try: + task = asyncio.create_task(coro) + tasks.add(task) + task.add_done_callback(lambda t: future.set_result(t.result())) + except Exception as e: + future.set_exception(e) + elif tasks: + done, tasks = await asyncio.wait( + tasks, return_when=asyncio.FIRST_COMPLETED + ) + for task in done: + if task.exception(): + logger.error(f"task exception: {task.exception()}") + future.set_exception(task.exception()) + else: + await asyncio.sleep(0.1) + logger.info("client __thread_routine end") + + async def submit_task(self, coro: Coroutine) -> Future: + future = Future() + await self.tasks.put((coro, future)) + return future + + def submit_task_with_new_thread(self, coro: Coroutine) -> Future: + future = Future() + + def run_coro_in_new_thread(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete(coro) + future.set_result(result) + except Exception as e: + future.set_exception(e) + finally: + loop.close() + + thread = threading.Thread(target=run_coro_in_new_thread) + thread.start() + return future diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/log.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/log.py new file mode 100644 index 00000000..0cfa1aaf --- /dev/null +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/log.py @@ -0,0 +1,13 @@ +import logging + +logger = logging.getLogger("aliyun_analyticdb_vector_storage") +logger.setLevel(logging.INFO) + +formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - [%(filename)s:%(lineno)d] - %(message)s" +) + +console_handler = logging.StreamHandler() +console_handler.setFormatter(formatter) + +logger.addHandler(console_handler) diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/manifest.json b/agents/addon/extension/aliyun_analyticdb_vector_storage/manifest.json new file mode 100644 index 00000000..c7a64f46 --- /dev/null +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/manifest.json @@ -0,0 +1,99 @@ +{ + "type": "extension", + "name": "aliyun_analyticdb_vector_storage", + "version": "0.4.0", + "language": "python", + "dependencies": [ + { + "type": "system", + "name": "rte_runtime_python", + "version": "0.4" + } + ], + "api": { + "property": { + "alibaba_cloud_access_key_id": { + "type": "string" + }, + "alibaba_cloud_access_key_secret": { + "type": "string" + }, + "adbpg_instance_id": { + "type": "string" + }, + "adbpg_instance_region": { + "type": "string" + }, + "adbpg_account": { + "type": "string" + }, + "adbpg_account_password": { + "type": "string" + }, + "adbpg_namespace": { + "type": "string" + }, + "adbpg_namespace_password": { + "type": "string" + } + }, + "cmd_in": [ + { + "name": "upsert_vector", + "property": { + "collection_name": { + "type": "string" + }, + "file_name": { + "type": "string" + }, + "content": { + "type": "string" + } + } + }, + { + "name": "query_vector", + "property": { + "collection_name": { + "type": "string" + }, + "top_k": { + "type": "int64" + }, + "embedding": { + "type": "array", + "items": { + "type": "float64" + } + } + } + }, + { + "name": "create_collection", + "property": { + "collection_name": { + "type": "string" + }, + "dimension": { + "type": "int32" + } + }, + "required": [ + "collection_name" + ] + }, + { + "name": "delete_collection", + "property": { + "collection_name": { + "type": "string" + } + }, + "required": [ + "collection_name" + ] + } + ] + } +} \ No newline at end of file diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/model.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/model.py new file mode 100644 index 00000000..4b3f4b12 --- /dev/null +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/model.py @@ -0,0 +1,547 @@ +# -*- coding: utf-8 -*- + +from alibabacloud_gpdb20160503 import models as gpdb_20160503_models # type: ignore + +try: + from .log import logger +except ImportError: + from log import logger +import time +import json +from typing import Dict, List, Any, Tuple +from alibabacloud_tea_util import models as util_models + + +class Model: + def __init__(self, region_id, dbinstance_id, client): + self.region_id = region_id + self.dbinstance_id = dbinstance_id + self.client = client + self.read_timeout = 10 * 1000 + self.connect_timeout = 10 * 1000 + + def get_client(self): + return self.client.get() + + def init_vector_database(self, account, account_password) -> None: + try: + request = gpdb_20160503_models.InitVectorDatabaseRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + manager_account=account, + manager_account_password=account_password, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = self.get_client().init_vector_database_with_options( + request, runtime + ) + logger.info( + f"init_vector_database response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + async def init_vector_database_async(self, account, account_password) -> None: + try: + request = gpdb_20160503_models.InitVectorDatabaseRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + manager_account=account, + manager_account_password=account_password, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = await self.get_client().init_vector_database_with_options_async( + request, runtime + ) + logger.info( + f"init_vector_database response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + def create_namespace( + self, account, account_password, namespace, namespace_password + ) -> None: + try: + request = gpdb_20160503_models.CreateNamespaceRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + manager_account=account, + manager_account_password=account_password, + namespace=namespace, + namespace_password=namespace_password, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = self.get_client().create_namespace_with_options(request, runtime) + logger.info( + f"create_namespace response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + async def create_namespace_async( + self, account, account_password, namespace, namespace_password + ) -> None: + try: + request = gpdb_20160503_models.CreateNamespaceRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + manager_account=account, + manager_account_password=account_password, + namespace=namespace, + namespace_password=namespace_password, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = await self.get_client().create_namespace_with_options_async( + request, runtime + ) + logger.info( + f"create_namespace response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + def create_collection( + self, + account, + account_password, + namespace, + collection, + parser: str = None, + metrics: str = None, + hnsw_m: int = None, + pq_enable: int = None, + external_storage: int = None, + ) -> None: + try: + metadata = '{"update_ts": "bigint", "file_name": "text", "content": "text"}' + full_text_retrieval_fields = "update_ts,file_name" + request = gpdb_20160503_models.CreateCollectionRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + manager_account=account, + manager_account_password=account_password, + namespace=namespace, + collection=collection, + metadata=metadata, + full_text_retrieval_fields=full_text_retrieval_fields, + parser=parser, + metrics=metrics, + hnsw_m=hnsw_m, + pq_enable=pq_enable, + external_storage=external_storage, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = self.get_client().create_collection_with_options( + request, runtime + ) + logger.info( + f"create_document_collection response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + async def create_collection_async( + self, + account, + account_password, + namespace, + collection, + parser: str = None, + metrics: str = None, + hnsw_m: int = None, + pq_enable: int = None, + external_storage: int = None, + ) -> None: + try: + metadata = '{"update_ts": "bigint", "file_name": "text", "content": "text"}' + full_text_retrieval_fields = "update_ts,file_name" + request = gpdb_20160503_models.CreateCollectionRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + manager_account=account, + manager_account_password=account_password, + namespace=namespace, + collection=collection, + metadata=metadata, + full_text_retrieval_fields=full_text_retrieval_fields, + parser=parser, + metrics=metrics, + hnsw_m=hnsw_m, + pq_enable=pq_enable, + external_storage=external_storage, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = await self.get_client().create_collection_with_options_async( + request, runtime + ) + logger.info( + f"create_document_collection response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + def delete_collection(self, namespace, namespace_password, collection) -> None: + try: + request = gpdb_20160503_models.DeleteCollectionRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + namespace_password=namespace_password, + namespace=namespace, + collection=collection, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = self.get_client().delete_collection_with_options( + request, runtime + ) + logger.info( + f"delete_collection response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + async def delete_collection_async( + self, namespace, namespace_password, collection + ) -> None: + try: + request = gpdb_20160503_models.DeleteCollectionRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + namespace_password=namespace_password, + namespace=namespace, + collection=collection, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = await self.get_client().delete_collection_with_options_async( + request, runtime + ) + logger.info( + f"delete_collection response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + def upsert_collection_data( + self, + collection, + namespace, + namespace_password, + rows: List[Tuple[str, str, List[float]]] = None, + ) -> None: + try: + request_rows = [] + for row in rows: + file_name = row[0] + content = row[1] + vector = row[2] + metadata = { + "update_ts": int(time.time() * 1000), + "file_name": file_name, + "content": content, + } + request_row = gpdb_20160503_models.UpsertCollectionDataRequestRows( + metadata=metadata, vector=vector + ) + request_rows.append(request_row) + upsert_collection_data_request = ( + gpdb_20160503_models.UpsertCollectionDataRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + collection=collection, + namespace_password=namespace_password, + namespace=namespace, + rows=request_rows, + ) + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = self.get_client().upsert_collection_data_with_options( + upsert_collection_data_request, runtime + ) + logger.info( + f"upsert_collection response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + async def upsert_collection_data_async( + self, + collection, + namespace, + namespace_password, + rows: List[Tuple[str, str, List[float]]] = None, + ) -> None: + try: + request_rows = [] + for row in rows: + file_name = row[0] + content = row[1] + vector = row[2] + metadata = { + "update_ts": int(time.time() * 1000), + "file_name": file_name, + "content": content, + } + request_row = gpdb_20160503_models.UpsertCollectionDataRequestRows( + metadata=metadata, vector=vector + ) + request_rows.append(request_row) + upsert_collection_data_request = ( + gpdb_20160503_models.UpsertCollectionDataRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + collection=collection, + namespace_password=namespace_password, + namespace=namespace, + rows=request_rows, + ) + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = ( + await self.get_client().upsert_collection_data_with_options_async( + upsert_collection_data_request, runtime + ) + ) + logger.info( + f"upsert_collection response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + def query_collection_data( + self, + collection, + namespace, + namespace_password, + vector: List[float] = None, + top_k: int = 10, + content: str = None, + filter: str = None, + hybrid_search: str = None, + hybrid_search_args: Dict[str, dict] = None, + include_metadata_fields: str = None, + include_values: bool = None, + metrics: str = None, + ) -> Tuple[Any, Any]: + try: + query_collection_data_request = ( + gpdb_20160503_models.QueryCollectionDataRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + collection=collection, + namespace_password=namespace_password, + namespace=namespace, + vector=vector, + top_k=top_k, + content=content, + filter=filter, + hybrid_search=hybrid_search, + hybrid_search_args=hybrid_search_args, + include_metadata_fields=include_metadata_fields, + include_values=include_values, + metrics=metrics, + ) + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = self.get_client().query_collection_data_with_options( + query_collection_data_request, runtime + ) + # logger.info(f"query_collection response code: {response.status_code}, body:{response.body}") + logger.info(f"query_collection response code: {response.status_code}") + return response, None + except Exception as e: + logger.error(f"Error: {e}") + return None, e + + async def query_collection_data_async( + self, + collection, + namespace, + namespace_password, + vector: List[float] = None, + top_k: int = 10, + content: str = None, + filter: str = None, + hybrid_search: str = None, + hybrid_search_args: Dict[str, dict] = None, + include_metadata_fields: str = None, + include_values: bool = None, + metrics: str = None, + ) -> Tuple[Any, Any]: + try: + query_collection_data_request = ( + gpdb_20160503_models.QueryCollectionDataRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + collection=collection, + namespace_password=namespace_password, + namespace=namespace, + vector=vector, + top_k=top_k, + content=content, + filter=filter, + hybrid_search=hybrid_search, + hybrid_search_args=hybrid_search_args, + include_metadata_fields=include_metadata_fields, + include_values=include_values, + metrics=metrics, + ) + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = await self.get_client().query_collection_data_with_options_async( + query_collection_data_request, runtime + ) + logger.info(f"query_collection response code: {response.status_code}") + return response, None + except Exception as e: + logger.error(f"Error: {e}") + return None, e + + def parse_collection_data( + self, body: gpdb_20160503_models.QueryCollectionDataResponseBody + ) -> str: + try: + matches = body.to_map()["Matches"]["match"] + results = [ + {"content": match["Metadata"]["content"], "score": match["Score"]} + for match in matches + ] + results.sort(key=lambda x: x["score"], reverse=True) + json_str = json.dumps(results) + return json_str + except Exception as e: + logger.error(f"Error: {e}") + return "[]" + + def list_collections(self, namespace, namespace_password) -> Tuple[List[str], Any]: + try: + request = gpdb_20160503_models.ListCollectionsRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + namespace=namespace, + namespace_password=namespace_password, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = self.get_client().list_collections_with_options(request, runtime) + logger.info( + f"list_collections response code: {response.status_code}, body:{response.body}" + ) + collections = response.body.to_map()["Collections"]["collection"] + return collections, None + except Exception as e: + logger.error(f"Error: {e}") + return [], e + + async def list_collections_async( + self, namespace, namespace_password + ) -> Tuple[List[str], Any]: + try: + request = gpdb_20160503_models.ListCollectionsRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + namespace=namespace, + namespace_password=namespace_password, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = await self.get_client().list_collections_with_options_async( + request, runtime + ) + logger.info( + f"list_collections response code: {response.status_code}, body:{response.body}" + ) + collections = response.body.to_map()["Collections"]["collection"] + return collections, None + except Exception as e: + logger.error(f"Error: {e}") + return [], e + + def create_vector_index( + self, account, account_password, namespace, collection, dimension + ) -> None: + try: + request = gpdb_20160503_models.CreateVectorIndexRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + manager_account=account, + manager_account_password=account_password, + namespace=namespace, + collection=collection, + dimension=dimension, + pq_enable=0, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = self.get_client().create_vector_index_with_options( + request, runtime + ) + logger.info( + f"create_vector_index response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e + + async def create_vector_index_async( + self, account, account_password, namespace, collection, dimension + ) -> None: + try: + request = gpdb_20160503_models.CreateVectorIndexRequest( + region_id=self.region_id, + dbinstance_id=self.dbinstance_id, + manager_account=account, + manager_account_password=account_password, + namespace=namespace, + collection=collection, + dimension=dimension, + pq_enable=0, + ) + runtime = util_models.RuntimeOptions( + read_timeout=self.read_timeout, connect_timeout=self.connect_timeout + ) + response = await self.get_client().create_vector_index_with_options_async( + request, runtime + ) + logger.info( + f"create_vector_index response code: {response.status_code}, body:{response.body}" + ) + except Exception as e: + logger.error(f"Error: {e}") + return e diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/property.json b/agents/addon/extension/aliyun_analyticdb_vector_storage/property.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/requirements.txt b/agents/addon/extension/aliyun_analyticdb_vector_storage/requirements.txt new file mode 100644 index 00000000..fa0bed40 --- /dev/null +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/requirements.txt @@ -0,0 +1 @@ +alibabacloud_gpdb20160503 \ No newline at end of file diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_addon.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_addon.py new file mode 100644 index 00000000..6bb26538 --- /dev/null +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_addon.py @@ -0,0 +1,14 @@ +from rte import ( + Addon, + register_addon_as_extension, + RteEnv, +) +from .log import logger +from .vector_storage_extension import AliPGDBExtension + + +@register_addon_as_extension("aliyun_analyticdb_vector_storage") +class AliPGDBExtensionAddon(Addon): + def on_create_instance(self, rte: RteEnv, addon_name: str, context) -> None: + logger.info("on_create_instance") + rte.on_create_instance_done(AliPGDBExtension(addon_name), context) diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py new file mode 100644 index 00000000..ea0daf81 --- /dev/null +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +# + +import asyncio +import os +import json +from .client import AliGPDBClient +from .model import Model +from rte import ( + Extension, + RteEnv, + Cmd, + Data, + StatusCode, + CmdResult, +) + +from typing import List +from .log import logger +import threading + +from alibabacloud_gpdb20160503.client import Client as gpdb20160503Client +from alibabacloud_tea_openapi import models as open_api_models +from alibabacloud_gpdb20160503 import models as gpdb_20160503_models +from alibabacloud_tea_util import models as util_models +from alibabacloud_tea_util.client import Client as UtilClient + + +class AliPGDBExtension(Extension): + def __init__(self, name): + self.stopEvent = asyncio.Event() + self.thread = None + self.loop = None + self.access_key_id = os.environ.get("ALIBABA_CLOUD_ACCESS_KEY_ID") + self.access_key_secret = os.environ.get("ALIBABA_CLOUD_ACCESS_KEY_SECRET") + self.region_id = os.environ.get("ADBPG_INSTANCE_REGION") + self.dbinstance_id = os.environ.get("ADBPG_INSTANCE_ID") + self.endpoint = f"gpdb.aliyuncs.com" + self.client = None + self.account = os.environ.get("ADBPG_ACCOUNT") + self.account_password = os.environ.get("ADBPG_ACCOUNT_PASSWORD") + self.namespace = os.environ.get("ADBPG_NAMESPACE") + self.namespace_password = os.environ.get("ADBPG_NAMESPACE_PASSWORD") + + async def __thread_routine(self, rte_env: RteEnv): + logger.info("__thread_routine start") + self.loop = asyncio.get_running_loop() + rte_env.on_start_done() + await self.stopEvent.wait() + + async def stop_thread(self): + self.stopEvent.set() + + def on_start(self, rte: RteEnv) -> None: + logger.info(f"on_start") + self.access_key_id = self.get_property_string( + rte, "ALIBABA_CLOUD_ACCESS_KEY_ID", self.access_key_id + ) + self.access_key_secret = self.get_property_string( + rte, "ALIBABA_CLOUD_ACCESS_KEY_SECRET", self.access_key_secret + ) + self.region_id = self.get_property_string( + rte, "ADBPG_INSTANCE_REGION", self.region_id + ) + self.dbinstance_id = self.get_property_string( + rte, "ADBPG_INSTANCE_ID", self.dbinstance_id + ) + self.account = self.get_property_string(rte, "ADBPG_ACCOUNT", self.account) + self.account_password = self.get_property_string( + rte, "ADBPG_ACCOUNT_PASSWORD", self.account_password + ) + self.namespace = self.get_property_string( + rte, "ADBPG_NAMESPACE", self.namespace + ) + self.namespace_password = self.get_property_string( + rte, "ADBPG_NAMESPACE_PASSWORD", self.namespace_password + ) + + if self.region_id in ( + "cn-beijing", + "cn-hangzhou", + "cn-shanghai", + "cn-shenzhen", + "cn-hongkong", + "ap-southeast-1", + "cn-hangzhou-finance", + "cn-shanghai-finance-1", + "cn-shenzhen-finance-1", + "cn-beijing-finance-1", + ): + self.endpoint = "gpdb.aliyuncs.com" + else: + self.endpoint = f"gpdb.{self.region_id}.aliyuncs.com" + self.client = AliGPDBClient( + self.access_key_id, self.access_key_secret, self.endpoint + ) + self.thread = threading.Thread( + target=asyncio.run, args=(self.__thread_routine(rte),) + ) + + # Then 'on_start_done' will be called in the thread + self.thread.start() + return + + def on_stop(self, rte: RteEnv) -> None: + logger.info("on_stop") + if self.thread is not None and self.thread.is_alive(): + asyncio.run_coroutine_threadsafe(self.stop_thread(), self.loop) + self.thread.join() + self.thread = None + rte.on_stop_done() + return + + def on_data(self, rte: RteEnv, data: Data) -> None: + pass + + def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: + try: + cmd_name = cmd.get_name() + logger.info(f"on_cmd [{cmd_name}]") + if cmd_name == "create_collection": + asyncio.run_coroutine_threadsafe( + self.async_create_collection(rte, cmd), self.loop + ) + elif cmd_name == "delete_collection": + asyncio.run_coroutine_threadsafe( + self.async_delete_collection(rte, cmd), self.loop + ) + elif cmd_name == "upsert_vector": + asyncio.run_coroutine_threadsafe( + self.async_upsert_vector(rte, cmd), self.loop + ) + elif cmd_name == "query_vector": + asyncio.run_coroutine_threadsafe( + self.async_query_vector(rte, cmd), self.loop + ) + else: + rte.return_result(CmdResult.create(StatusCode.ERROR), cmd) + except Exception as e: + rte.return_result(CmdResult.create(StatusCode.ERROR), cmd) + + async def async_create_collection(self, rte: RteEnv, cmd: Cmd): + m = Model(self.region_id, self.dbinstance_id, self.client) + collection = cmd.get_property_string("collection_name") + dimension = cmd.get_property_int("dimension") + if not dimension: + dimension = 1024 + err = await m.create_collection_async( + self.account, self.account_password, self.namespace, collection + ) + if err is None: + await m.create_vector_index_async( + self.account, + self.account_password, + self.namespace, + collection, + dimension, + ) + rte.return_result(CmdResult.create(StatusCode.OK), cmd) + else: + rte.return_result(CmdResult.create(StatusCode.ERROR), cmd) + + async def async_upsert_vector(self, rte: RteEnv, cmd: Cmd): + m = Model(self.region_id, self.dbinstance_id, self.client) + collection = cmd.get_property_string("collection_name") + file = cmd.get_property_string("file_name") + content = cmd.get_property_string("content") + obj = json.loads(content) + rows = [(file, item["text"], item["embedding"]) for item in obj] + # TODO remove the following line when create_collection is implemented + await m.create_collection_async( + self.account, self.account_password, self.namespace, collection + ) + err = await m.upsert_collection_data_async( + collection, self.namespace, self.namespace_password, rows + ) + if err is None: + rte.return_result(CmdResult.create(StatusCode.OK), cmd) + else: + rte.return_result(CmdResult.create(StatusCode.ERROR), cmd) + + async def async_query_vector(self, rte: RteEnv, cmd: Cmd): + m = Model(self.region_id, self.dbinstance_id, self.client) + collection = cmd.get_property_string("collection_name") + embedding = cmd.get_property_to_json("embedding") + top_k = cmd.get_property_int("top_k") + vector = json.loads(embedding) + response, error = await m.query_collection_data_async( + collection, self.namespace, self.namespace_password, vector, top_k=top_k + ) + if error: + return rte.return_result(CmdResult.create(StatusCode.ERROR), cmd) + else: + body = m.parse_collection_data(response.body) + ret = CmdResult.create(StatusCode.OK) + ret.set_property_from_json("response", body) + rte.return_result(ret, cmd) + + async def async_delete_collection(self, rte: RteEnv, cmd: Cmd): + m = Model(self.region_id, self.dbinstance_id, self.client) + collection = cmd.get_property_string("collection_name") + err = await m.delete_collection_async( + self.account, self.account_password, self.namespace, collection + ) + if err is None: + return rte.return_result(CmdResult.create(StatusCode.OK), cmd) + else: + return rte.return_result(CmdResult.create(StatusCode.ERROR), cmd) + + def get_property_string(self, rte: RteEnv, key: str, default: str) -> str: + try: + return rte.get_property_string(key.lower()) + except Exception as e: + logger.error(f"Error: {e}") + return default From 4a88988aeb290782891c6180cf42e421bbddec35 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Wed, 7 Aug 2024 01:22:30 -0700 Subject: [PATCH 04/34] feat: add file_chunker --- .../addon/extension/file_chunker/__init__.py | 4 + .../file_chunker/file_chunker_addon.py | 14 ++ .../file_chunker/file_chunker_extension.py | 207 ++++++++++++++++++ agents/addon/extension/file_chunker/log.py | 13 ++ .../extension/file_chunker/manifest.json | 80 +++++++ .../extension/file_chunker/property.json | 1 + .../extension/file_chunker/requirements.txt | 2 + 7 files changed, 321 insertions(+) create mode 100644 agents/addon/extension/file_chunker/__init__.py create mode 100644 agents/addon/extension/file_chunker/file_chunker_addon.py create mode 100644 agents/addon/extension/file_chunker/file_chunker_extension.py create mode 100644 agents/addon/extension/file_chunker/log.py create mode 100644 agents/addon/extension/file_chunker/manifest.json create mode 100644 agents/addon/extension/file_chunker/property.json create mode 100644 agents/addon/extension/file_chunker/requirements.txt diff --git a/agents/addon/extension/file_chunker/__init__.py b/agents/addon/extension/file_chunker/__init__.py new file mode 100644 index 00000000..06d98178 --- /dev/null +++ b/agents/addon/extension/file_chunker/__init__.py @@ -0,0 +1,4 @@ +from . import file_chunker_addon +from .log import logger + +logger.info("file_chunker extension loaded") diff --git a/agents/addon/extension/file_chunker/file_chunker_addon.py b/agents/addon/extension/file_chunker/file_chunker_addon.py new file mode 100644 index 00000000..b5daf556 --- /dev/null +++ b/agents/addon/extension/file_chunker/file_chunker_addon.py @@ -0,0 +1,14 @@ +from rte import ( + Addon, + register_addon_as_extension, + RteEnv, +) +from .log import logger +from .file_chunker_extension import FileChunkerExtension + + +@register_addon_as_extension("file_chunker") +class FileChunkerExtensionAddon(Addon): + def on_create_instance(self, rte: RteEnv, addon_name: str, context) -> None: + logger.info("on_create_instance") + rte.on_create_instance_done(FileChunkerExtension(addon_name), context) diff --git a/agents/addon/extension/file_chunker/file_chunker_extension.py b/agents/addon/extension/file_chunker/file_chunker_extension.py new file mode 100644 index 00000000..00d5e20c --- /dev/null +++ b/agents/addon/extension/file_chunker/file_chunker_extension.py @@ -0,0 +1,207 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-05. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from rte import ( + Extension, + RteEnv, + Cmd, + StatusCode, + CmdResult, +) +from typing import List, Any +from .log import logger +from llama_index.core import SimpleDirectoryReader +from llama_index.core.node_parser import SentenceSplitter +import json +from datetime import datetime +import uuid, math +import queue, threading + +CMD_FILE_DOWNLOADED = "file_downloaded" +UPSERT_VECTOR_CMD = "upsert_vector" +FILE_CHUNKED_CMD = "file_chunked" +CHUNK_SIZE = 200 +CHUNK_OVERLAP = 20 +BATCH_SIZE = 5 + + +def batch(nodes, size): + batch_texts = [] + for n in nodes: + batch_texts.append(n.text) + if len(batch_texts) == size: + yield batch_texts[:] + batch_texts.clear() + if batch_texts: + yield batch_texts + + +class FileChunkerExtension(Extension): + def __init__(self, name: str): + super().__init__(name) + + self.counters = {} + self.expected = {} + self.new_collection_name = "" + self.file_chunked_event = threading.Event() + + self.thread = None + self.queue = queue.Queue() + self.stop = False + + def generate_collection_name(self) -> str: + """ + follow rules: ^[a-z]+[a-z0-9_]* + """ + + return "coll_" + uuid.uuid1().hex.lower() + + def split(self, path: str) -> List[Any]: + + # load pdf file by path + documents = SimpleDirectoryReader( + input_files=[path], filename_as_id=True + ).load_data() + + # split pdf file into chunks + splitter = SentenceSplitter( + chunk_size=CHUNK_SIZE, + chunk_overlap=CHUNK_OVERLAP, + ) + nodes = splitter.get_nodes_from_documents(documents) + logger.info("pages in pdf: {}, chunking count: {}", len(documents), len(nodes)) + return nodes + + def embedding(self, rte: RteEnv, path: str, texts: List[str]): + logger.info( + "generate embeddings for the file: {}, with batch size: {}".format( + path, len(texts) + ) + ) + + cmd_out = Cmd.create("embed_batch") + cmd_out.set_property_from_json("inputs", json.dumps(texts)) + rte.send_cmd( + cmd_out, lambda rte, result: self.vector_store(rte, path, texts, result) + ) + + def vector_store(self, rte: RteEnv, path: str, texts: List[str], result: CmdResult): + logger.info("vector store start for file {}".format(path)) + file_name = path.split("/")[-1] + embedOutputJson = result.get_property_string("output") + embedOutput = json.loads(embedOutputJson) + cmd_out = Cmd.create(UPSERT_VECTOR_CMD) + cmd_out.set_property_string("collection_name", self.new_collection_name) + cmd_out.set_property_string("file_name", file_name) + embeddings = [record["embedding"] for record in embedOutput["embeddings"]] + content = [] + for text, embedding in zip(texts, embeddings): + content.append({"text": text, "embedding": embedding}) + cmd_out.set_property_string("content", json.dumps(content)) + rte.send_cmd(cmd_out, lambda rte, result: self.file_chunked(rte, path)) + + def file_chunked(self, rte: RteEnv, path: str): + if path in self.counters and path in self.expected: + self.counters[path] += 1 + logger.info( + "complete vector store for one splitting of the file: %s, current counter: %i, expected: %i", + path, + self.counters[path], + self.expected[path], + ) + if self.counters[path] == self.expected[path]: + chunks_count = self.counters[path] + del self.counters[path] + del self.expected[path] + logger.info( + "complete chunk for the file: {}, chunks_count {}, latency {}ms ".format( + path, + chunks_count, + int((datetime.now() - self.start_time).total_seconds() * 1000), + ) + ) + cmd_out = Cmd.create(FILE_CHUNKED_CMD) + cmd_out.set_property_string("path", path) + cmd_out.set_property_string("collection_name", self.new_collection_name) + rte.send_cmd( + cmd_out, + lambda rte, result: logger.info("send_cmd done"), + ) + self.file_chunked_event.set() + else: + logger.error("missing counter for the file path: %s", path) + + def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: + cmd_name = cmd.get_name() + if cmd_name == CMD_FILE_DOWNLOADED: + path = cmd.get_property_string("path") + self.queue.put(path) # make sure files are processed in order + else: + logger.info("unknown cmd {}".format(cmd_name)) + + cmd_result = CmdResult.create(StatusCode.OK) + cmd_result.set_property_string("detail", "ok") + rte.return_result(cmd_result, cmd) + + def async_handler(self, rte: RteEnv) -> None: + while not self.stop: + path = self.queue.get() + if path is None: + break + + # start processing the file + start_time = datetime.now() + collection_name = self.generate_collection_name() + logger.info( + "start processing {}, collection_name {}".format(path, collection_name) + ) + + # split + nodes = self.split(path) + + # reset counters and events + self.new_collection_name = collection_name + self.expected[path] = math.ceil(len(nodes) / BATCH_SIZE) + self.counters[path] = 0 + self.file_chunked_event.clear() + + # trigger embedding and vector storing in parallel + for texts in list(batch(nodes, BATCH_SIZE)): + self.embedding(rte, path, texts) + + # wait for all chunks to be processed + self.file_chunked_event.wait() + + logger.info( + "finished processing {}, collection_name {}, cost {}ms".format( + path, + collection_name, + (datetime.now() - start_time).microseconds / 1000, + ) + ) + + def on_start(self, rte: RteEnv) -> None: + logger.info("on_start") + + self.stop = False + self.thread = threading.Thread(target=self.async_handler, args=[rte]) + self.thread.start() + + rte.on_start_done() + + def on_stop(self, rte: RteEnv) -> None: + logger.info("on_stop") + + self.stop = True + if self.thread is not None: + while not self.queue.empty(): + self.queue.get() + self.queue.put(None) + self.thread.join() + self.thread = None + + rte.on_stop_done() diff --git a/agents/addon/extension/file_chunker/log.py b/agents/addon/extension/file_chunker/log.py new file mode 100644 index 00000000..d9cb27d3 --- /dev/null +++ b/agents/addon/extension/file_chunker/log.py @@ -0,0 +1,13 @@ +import logging + +logger = logging.getLogger("file_chunker") +logger.setLevel(logging.INFO) + +formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - [%(filename)s:%(lineno)d] - %(message)s" +) + +console_handler = logging.StreamHandler() +console_handler.setFormatter(formatter) + +logger.addHandler(console_handler) diff --git a/agents/addon/extension/file_chunker/manifest.json b/agents/addon/extension/file_chunker/manifest.json new file mode 100644 index 00000000..79c42b2d --- /dev/null +++ b/agents/addon/extension/file_chunker/manifest.json @@ -0,0 +1,80 @@ +{ + "type": "extension", + "name": "file_chunker", + "version": "0.4.0", + "language": "python", + "dependencies": [ + { + "type": "system", + "name": "rte_runtime_python", + "version": "0.4" + } + ], + "api": { + "property": {}, + "cmd_in": [ + { + "name": "file_downloaded", + "property": { + "path": { + "type": "string" + } + }, + "required": [ + "path" + ] + } + ], + "cmd_out": [ + { + "name": "embed_batch", + "property": { + "inputs": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "result": { + "property": { + "output": { + "type": "string" + }, + "code": { + "type": "string" + }, + "message": { + "type": "string" + } + } + } + }, + { + "name": "upsert_vector", + "property": { + "collection_name": { + "type": "string" + }, + "file_name": { + "type": "string" + }, + "content": { + "type": "string" + } + } + }, + { + "name": "file_chunked", + "property": { + "path": { + "type": "string" + }, + "collection_name": { + "type": "string" + } + } + } + ] + } +} \ No newline at end of file diff --git a/agents/addon/extension/file_chunker/property.json b/agents/addon/extension/file_chunker/property.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/agents/addon/extension/file_chunker/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/agents/addon/extension/file_chunker/requirements.txt b/agents/addon/extension/file_chunker/requirements.txt new file mode 100644 index 00000000..b5a22c29 --- /dev/null +++ b/agents/addon/extension/file_chunker/requirements.txt @@ -0,0 +1,2 @@ +pypdf +llama-index \ No newline at end of file From 7557830cfa64365d4040039ecb527a5bd72c37a6 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Wed, 7 Aug 2024 04:08:39 -0700 Subject: [PATCH 05/34] feat: file chunking --- .../aliyun_analyticdb_vector_storage/model.py | 30 +++++++++---------- .../vector_storage_extension.py | 11 +++++++ .../embedding_extension.py | 11 +++++-- .../aliyun_text_embedding/manifest.json | 19 ++---------- .../file_chunker/file_chunker_extension.py | 27 +++++++++++------ .../extension/file_chunker/manifest.json | 24 +++++++++------ 6 files changed, 69 insertions(+), 53 deletions(-) diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/model.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/model.py index 4b3f4b12..48365736 100644 --- a/agents/addon/extension/aliyun_analyticdb_vector_storage/model.py +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/model.py @@ -37,7 +37,7 @@ def init_vector_database(self, account, account_password) -> None: response = self.get_client().init_vector_database_with_options( request, runtime ) - logger.info( + logger.debug( f"init_vector_database response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -58,7 +58,7 @@ async def init_vector_database_async(self, account, account_password) -> None: response = await self.get_client().init_vector_database_with_options_async( request, runtime ) - logger.info( + logger.debug( f"init_vector_database response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -81,7 +81,7 @@ def create_namespace( read_timeout=self.read_timeout, connect_timeout=self.connect_timeout ) response = self.get_client().create_namespace_with_options(request, runtime) - logger.info( + logger.debug( f"create_namespace response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -106,7 +106,7 @@ async def create_namespace_async( response = await self.get_client().create_namespace_with_options_async( request, runtime ) - logger.info( + logger.debug( f"create_namespace response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -149,7 +149,7 @@ def create_collection( response = self.get_client().create_collection_with_options( request, runtime ) - logger.info( + logger.debug( f"create_document_collection response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -192,7 +192,7 @@ async def create_collection_async( response = await self.get_client().create_collection_with_options_async( request, runtime ) - logger.info( + logger.debug( f"create_document_collection response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -214,7 +214,7 @@ def delete_collection(self, namespace, namespace_password, collection) -> None: response = self.get_client().delete_collection_with_options( request, runtime ) - logger.info( + logger.debug( f"delete_collection response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -283,7 +283,7 @@ def upsert_collection_data( response = self.get_client().upsert_collection_data_with_options( upsert_collection_data_request, runtime ) - logger.info( + logger.debug( f"upsert_collection response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -330,7 +330,7 @@ async def upsert_collection_data_async( upsert_collection_data_request, runtime ) ) - logger.info( + logger.debug( f"upsert_collection response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -378,7 +378,7 @@ def query_collection_data( query_collection_data_request, runtime ) # logger.info(f"query_collection response code: {response.status_code}, body:{response.body}") - logger.info(f"query_collection response code: {response.status_code}") + logger.debug(f"query_collection response code: {response.status_code}") return response, None except Exception as e: logger.error(f"Error: {e}") @@ -424,7 +424,7 @@ async def query_collection_data_async( response = await self.get_client().query_collection_data_with_options_async( query_collection_data_request, runtime ) - logger.info(f"query_collection response code: {response.status_code}") + logger.debug(f"query_collection response code: {response.status_code}") return response, None except Exception as e: logger.error(f"Error: {e}") @@ -458,7 +458,7 @@ def list_collections(self, namespace, namespace_password) -> Tuple[List[str], An read_timeout=self.read_timeout, connect_timeout=self.connect_timeout ) response = self.get_client().list_collections_with_options(request, runtime) - logger.info( + logger.debug( f"list_collections response code: {response.status_code}, body:{response.body}" ) collections = response.body.to_map()["Collections"]["collection"] @@ -483,7 +483,7 @@ async def list_collections_async( response = await self.get_client().list_collections_with_options_async( request, runtime ) - logger.info( + logger.debug( f"list_collections response code: {response.status_code}, body:{response.body}" ) collections = response.body.to_map()["Collections"]["collection"] @@ -512,7 +512,7 @@ def create_vector_index( response = self.get_client().create_vector_index_with_options( request, runtime ) - logger.info( + logger.debug( f"create_vector_index response code: {response.status_code}, body:{response.body}" ) except Exception as e: @@ -539,7 +539,7 @@ async def create_vector_index_async( response = await self.get_client().create_vector_index_with_options_async( request, runtime ) - logger.info( + logger.debug( f"create_vector_index response code: {response.status_code}, body:{response.body}" ) except Exception as e: diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py index ea0daf81..f582e18a 100644 --- a/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py @@ -18,6 +18,7 @@ from typing import List from .log import logger import threading +from datetime import datetime from alibabacloud_gpdb20160503.client import Client as gpdb20160503Client from alibabacloud_tea_openapi import models as open_api_models @@ -161,6 +162,7 @@ async def async_create_collection(self, rte: RteEnv, cmd: Cmd): rte.return_result(CmdResult.create(StatusCode.ERROR), cmd) async def async_upsert_vector(self, rte: RteEnv, cmd: Cmd): + start_time = datetime.now() m = Model(self.region_id, self.dbinstance_id, self.client) collection = cmd.get_property_string("collection_name") file = cmd.get_property_string("file_name") @@ -174,6 +176,15 @@ async def async_upsert_vector(self, rte: RteEnv, cmd: Cmd): err = await m.upsert_collection_data_async( collection, self.namespace, self.namespace_password, rows ) + logger.info( + "upsert_vector finished for file {}, collection {}, rows len {}, err {}, cost {}ms".format( + file, + collection, + len(rows), + err, + int((datetime.now() - start_time).total_seconds() * 1000), + ) + ) if err is None: rte.return_result(CmdResult.create(StatusCode.OK), cmd) else: diff --git a/agents/addon/extension/aliyun_text_embedding/embedding_extension.py b/agents/addon/extension/aliyun_text_embedding/embedding_extension.py index 08077618..ddc74710 100644 --- a/agents/addon/extension/aliyun_text_embedding/embedding_extension.py +++ b/agents/addon/extension/aliyun_text_embedding/embedding_extension.py @@ -63,6 +63,7 @@ def async_handler(self, index: int, rte: RteEnv): break cmd_name = cmd.get_name() + start_time = datetime.now() logger.info( "async_handler {} processing cmd {}".format(index, cmd_name)) @@ -76,12 +77,15 @@ def async_handler(self, index: int, rte: RteEnv): else: logger.warning("unknown cmd {}".format(cmd_name)) + logger.info( + "async_handler {} finished processing cmd {}, cost {}ms".format(index, cmd_name, int((datetime.now() - start_time).total_seconds() * 1000))) + logger.info("async_handler {} stopped".format(index)) def call_with_str(self, message: str) -> CmdResult: start_time = datetime.now() response = dashscope.TextEmbedding.call(model=self.model, input=message) - logger.info("embedding call finished for input [{}], status_code {}, cost {}ms".format(message, response.status_code, (datetime.now() - start_time).microseconds / 1000)) + logger.info("embedding call finished for input [{}], status_code {}, cost {}ms".format(message, response.status_code, int((datetime.now() - start_time).total_seconds() * 1000))) if response.status_code == HTTPStatus.OK: cmd_result = CmdResult.create(StatusCode.OK) @@ -117,14 +121,15 @@ def call_with_strs(self, messages: List[str]) -> CmdResult: logger.error("call %s failed, errmsg: %s", batch, response) batch_counter += len(batch) - logger.info("embedding call finished for inputs len {}, batch_counter {}, results len {}, cost {}ms ".format(len(messages), batch_counter, len(result["embeddings"]), (datetime.now() - start_time).microseconds / 1000)) + logger.info("embedding call finished for inputs len {}, batch_counter {}, results len {}, cost {}ms ".format(len(messages), batch_counter, len(result["embeddings"]), int((datetime.now() - start_time).total_seconds() * 1000))) if result is not None: cmd_result = CmdResult.create(StatusCode.OK) - cmd_result.set_property_from_json(FIELD_KEY_EMBEDDINGS, result["embeddings"]) + cmd_result.set_property_string(FIELD_KEY_EMBEDDINGS, json.dumps(result["embeddings"])) return cmd_result else: cmd_result = CmdResult.create(StatusCode.ERROR) cmd_result.set_property_string(FIELD_KEY_MESSAGE, "All batch failed") + logger.error("All batch failed") return cmd_result def on_stop(self, rte: RteEnv) -> None: diff --git a/agents/addon/extension/aliyun_text_embedding/manifest.json b/agents/addon/extension/aliyun_text_embedding/manifest.json index ff9c46a2..2fd19099 100644 --- a/agents/addon/extension/aliyun_text_embedding/manifest.json +++ b/agents/addon/extension/aliyun_text_embedding/manifest.json @@ -63,21 +63,7 @@ "result": { "property": { "embeddings": { - "type": "array", - "items": { - "type": "object", - "properties": { - "embedding": { - "type": "array", - "items": { - "type": "float64" - } - }, - "text_index": { - "type": "int32" - } - } - } + "type": "string" }, "code": { "type": "string" @@ -88,7 +74,6 @@ } } } - ], - "cmd_out": [] + ] } } \ No newline at end of file diff --git a/agents/addon/extension/file_chunker/file_chunker_extension.py b/agents/addon/extension/file_chunker/file_chunker_extension.py index 00d5e20c..a0f4b531 100644 --- a/agents/addon/extension/file_chunker/file_chunker_extension.py +++ b/agents/addon/extension/file_chunker/file_chunker_extension.py @@ -24,6 +24,8 @@ CMD_FILE_DOWNLOADED = "file_downloaded" UPSERT_VECTOR_CMD = "upsert_vector" FILE_CHUNKED_CMD = "file_chunked" + +# TODO: configable CHUNK_SIZE = 200 CHUNK_OVERLAP = 20 BATCH_SIZE = 5 @@ -73,7 +75,11 @@ def split(self, path: str) -> List[Any]: chunk_overlap=CHUNK_OVERLAP, ) nodes = splitter.get_nodes_from_documents(documents) - logger.info("pages in pdf: {}, chunking count: {}", len(documents), len(nodes)) + logger.info( + "file {} pages count {}, chunking count {}".format( + path, len(documents), len(nodes) + ) + ) return nodes def embedding(self, rte: RteEnv, path: str, texts: List[str]): @@ -86,22 +92,26 @@ def embedding(self, rte: RteEnv, path: str, texts: List[str]): cmd_out = Cmd.create("embed_batch") cmd_out.set_property_from_json("inputs", json.dumps(texts)) rte.send_cmd( - cmd_out, lambda rte, result: self.vector_store(rte, path, texts, result) + cmd_out, + lambda rte, result: self.vector_store( + rte, path, texts, result + ), # TODO: deal with error ) def vector_store(self, rte: RteEnv, path: str, texts: List[str], result: CmdResult): - logger.info("vector store start for file {}".format(path)) + logger.info("vector store start for one splitting of the file {}".format(path)) file_name = path.split("/")[-1] - embedOutputJson = result.get_property_string("output") - embedOutput = json.loads(embedOutputJson) + embed_output_json = result.get_property_string("embeddings") + embed_output = json.loads(embed_output_json) cmd_out = Cmd.create(UPSERT_VECTOR_CMD) cmd_out.set_property_string("collection_name", self.new_collection_name) cmd_out.set_property_string("file_name", file_name) - embeddings = [record["embedding"] for record in embedOutput["embeddings"]] + embeddings = [record["embedding"] for record in embed_output] content = [] for text, embedding in zip(texts, embeddings): content.append({"text": text, "embedding": embedding}) cmd_out.set_property_string("content", json.dumps(content)) + # logger.info(json.dumps(content)) rte.send_cmd(cmd_out, lambda rte, result: self.file_chunked(rte, path)) def file_chunked(self, rte: RteEnv, path: str): @@ -118,10 +128,9 @@ def file_chunked(self, rte: RteEnv, path: str): del self.counters[path] del self.expected[path] logger.info( - "complete chunk for the file: {}, chunks_count {}, latency {}ms ".format( + "complete chunk for the file: {}, chunks_count {}".format( path, chunks_count, - int((datetime.now() - self.start_time).total_seconds() * 1000), ) ) cmd_out = Cmd.create(FILE_CHUNKED_CMD) @@ -180,7 +189,7 @@ def async_handler(self, rte: RteEnv) -> None: "finished processing {}, collection_name {}, cost {}ms".format( path, collection_name, - (datetime.now() - start_time).microseconds / 1000, + int((datetime.now() - start_time).total_seconds() * 1000), ) ) diff --git a/agents/addon/extension/file_chunker/manifest.json b/agents/addon/extension/file_chunker/manifest.json index 79c42b2d..ed7eb072 100644 --- a/agents/addon/extension/file_chunker/manifest.json +++ b/agents/addon/extension/file_chunker/manifest.json @@ -36,15 +36,12 @@ } } }, + "required": [ + "inputs" + ], "result": { "property": { - "output": { - "type": "string" - }, - "code": { - "type": "string" - }, - "message": { + "embeddings": { "type": "string" } } @@ -62,7 +59,12 @@ "content": { "type": "string" } - } + }, + "required": [ + "collection_name", + "file_name", + "content" + ] }, { "name": "file_chunked", @@ -73,7 +75,11 @@ "collection_name": { "type": "string" } - } + }, + "required": [ + "path", + "collection_name" + ] } ] } From 8f5183ea937f18f1e3e3a9e6244d12bb4d86a9cb Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Wed, 7 Aug 2024 04:25:25 -0700 Subject: [PATCH 06/34] feat: create collection by file_chunker --- .../vector_storage_extension.py | 14 +++++++------- .../file_chunker/file_chunker_extension.py | 16 ++++++++++++++++ .../addon/extension/file_chunker/manifest.json | 14 ++++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py index f582e18a..5cb53fe2 100644 --- a/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py @@ -143,9 +143,12 @@ def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: async def async_create_collection(self, rte: RteEnv, cmd: Cmd): m = Model(self.region_id, self.dbinstance_id, self.client) collection = cmd.get_property_string("collection_name") - dimension = cmd.get_property_int("dimension") - if not dimension: - dimension = 1024 + dimension = 1024 + try: + dimension = cmd.get_property_int("dimension") + except Exception as e: + logger.warning(f"Error: {e}") + err = await m.create_collection_async( self.account, self.account_password, self.namespace, collection ) @@ -169,10 +172,7 @@ async def async_upsert_vector(self, rte: RteEnv, cmd: Cmd): content = cmd.get_property_string("content") obj = json.loads(content) rows = [(file, item["text"], item["embedding"]) for item in obj] - # TODO remove the following line when create_collection is implemented - await m.create_collection_async( - self.account, self.account_password, self.namespace, collection - ) + err = await m.upsert_collection_data_async( collection, self.namespace, self.namespace_password, rows ) diff --git a/agents/addon/extension/file_chunker/file_chunker_extension.py b/agents/addon/extension/file_chunker/file_chunker_extension.py index a0f4b531..d1fb4e02 100644 --- a/agents/addon/extension/file_chunker/file_chunker_extension.py +++ b/agents/addon/extension/file_chunker/file_chunker_extension.py @@ -82,6 +82,18 @@ def split(self, path: str) -> List[Any]: ) return nodes + def create_collection(self, rte: RteEnv, collection_name: str, wait: bool): + cmd_out = Cmd.create("create_collection") + cmd_out.set_property_string("collection_name", collection_name) + + wait_event = threading.Event() + rte.send_cmd( + cmd_out, + lambda rte, result: wait_event.set(), + ) + if wait: + wait_event.wait() + def embedding(self, rte: RteEnv, path: str, texts: List[str]): logger.info( "generate embeddings for the file: {}, with batch size: {}".format( @@ -169,6 +181,10 @@ def async_handler(self, rte: RteEnv) -> None: "start processing {}, collection_name {}".format(path, collection_name) ) + # create collection + self.create_collection(rte, collection_name, True) + logger.info("collection_name {} created".format(collection_name)) + # split nodes = self.split(path) diff --git a/agents/addon/extension/file_chunker/manifest.json b/agents/addon/extension/file_chunker/manifest.json index ed7eb072..248114db 100644 --- a/agents/addon/extension/file_chunker/manifest.json +++ b/agents/addon/extension/file_chunker/manifest.json @@ -66,6 +66,20 @@ "content" ] }, + { + "name": "create_collection", + "property": { + "collection_name": { + "type": "string" + }, + "dimension": { + "type": "int32" + } + }, + "required": [ + "collection_name" + ] + }, { "name": "file_chunked", "property": { From 447806cc7fba673d91d715cc36fd648d5544c9aa Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Wed, 7 Aug 2024 05:25:19 -0700 Subject: [PATCH 07/34] fix: typo --- .../extension/aliyun_text_embedding/embedding_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/addon/extension/aliyun_text_embedding/embedding_extension.py b/agents/addon/extension/aliyun_text_embedding/embedding_extension.py index ddc74710..f3556c4c 100644 --- a/agents/addon/extension/aliyun_text_embedding/embedding_extension.py +++ b/agents/addon/extension/aliyun_text_embedding/embedding_extension.py @@ -35,7 +35,7 @@ def __init__(self, name: str): self.queue = queue.Queue() self.threads = [] - # TODO: workaround to speech up the embedding process, + # TODO: workaround to speed up the embedding process, # should be replace by https://help.aliyun.com/zh/model-studio/developer-reference/text-embedding-batch-api?spm=a2c4g.11186623.0.0.24cb7453KSjdhC # once v3 models supported self.parallel = 10 From 68e65d09278168c1a729f0b05282ad30dea06055 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Wed, 7 Aug 2024 23:46:19 -0700 Subject: [PATCH 08/34] feat: add call_chat api --- .../extension/qwen_llm_python/__init__.py | 3 +- .../extension/qwen_llm_python/manifest.json | 28 ++- .../qwen_llm_python/qwen_llm_extension.py | 181 ++++++++++-------- 3 files changed, 130 insertions(+), 82 deletions(-) diff --git a/agents/addon/extension/qwen_llm_python/__init__.py b/agents/addon/extension/qwen_llm_python/__init__.py index b5b5c7c2..3d3b6a9c 100644 --- a/agents/addon/extension/qwen_llm_python/__init__.py +++ b/agents/addon/extension/qwen_llm_python/__init__.py @@ -1,3 +1,4 @@ from . import qwen_llm_addon +from .log import logger -print("qwen_llm_python extension loaded") +logger.info("qwen_llm_python extension loaded") diff --git a/agents/addon/extension/qwen_llm_python/manifest.json b/agents/addon/extension/qwen_llm_python/manifest.json index 097d87b4..153369c1 100644 --- a/agents/addon/extension/qwen_llm_python/manifest.json +++ b/agents/addon/extension/qwen_llm_python/manifest.json @@ -1,13 +1,13 @@ { "type": "extension", "name": "qwen_llm_python", - "version": "0.1.0", + "version": "0.4.0", "language": "python", "dependencies": [ { "type": "system", "name": "rte_runtime_python", - "version": "0.4.0" + "version": "0.4" } ], "api": { @@ -60,6 +60,30 @@ "cmd_in": [ { "name": "flush" + }, + { + "name": "call_chat", + "property": { + "messages": { + "type": "string" + }, + "stream": { + "type": "bool" + } + }, + "required": [ + "messages" + ], + "result": { + "property": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ] + } } ], "cmd_out": [ diff --git a/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py b/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py index bcb7612b..2094d4e8 100644 --- a/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py +++ b/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py @@ -12,11 +12,11 @@ Data, StatusCode, CmdResult, - MetadataInfo, ) from typing import List, Any import dashscope import queue +import json from datetime import datetime import threading from http import HTTPStatus @@ -47,8 +47,7 @@ def __init__(self, name: str): self.max_history = 10 self.stopped = False self.thread = None - self.outdateTs = datetime.now() - self.ongoing = "" + self.outdate_ts = datetime.now() self.queue = queue.Queue() self.mutex = threading.Lock() @@ -75,38 +74,63 @@ def get_messages(self) -> List[Any]: return messages def need_interrupt(self, ts: datetime.time) -> bool: - return self.outdateTs > ts and (self.outdateTs - ts).total_seconds() > 1 + return self.outdate_ts > ts and (self.outdate_ts - ts).total_seconds() > 1 - def call(self, messages: List[Any]): - logger.info("before call %s", messages) - response = dashscope.Generation.call( - "qwen-max", - messages=messages, - result_format="message", # set the result to be "message" format. - stream=False, # set streaming output - incremental_output=False, # get streaming output incrementally - ) - if response.status_code == HTTPStatus.OK: - self.on_msg( - response.output.choices[0]["message"]["role"], - response.output.choices[0]["message"]["content"], - ) - logger.info( - "on response %s", response.output.choices[0]["message"]["content"] - ) + def complete_with_history(self, rte: RteEnv, ts: datetime.time, input_text: str): + """ + Complete input_text querying with built-in chat history. + """ + + def callback(text: str, end_of_segment: bool): + d = Data.create("text_data") + d.set_property_string("text", text) + d.set_property_bool("end_of_segment", end_of_segment) + rte.send_data(d) + + messages = self.get_messages() + messages.append({"role": "user", "content": input_text}) + total = self.stream_chat(ts, messages, callback) + self.on_msg("user", input_text) + if len(total) > 0: + self.on_msg("assistant", total) + + def call_chat(self, rte: RteEnv, ts: datetime.time, cmd: Cmd): + """ + Respond to call_chat cmd and return results in streaming. + The incoming 'messages' will contains all the system prompt, chat history and question. + """ + + def callback(text: str, end_of_segment: bool): + cmd_result = CmdResult.create(StatusCode.OK) + cmd_result.set_property_string("text", text) + if end_of_segment: + cmd_result.set_is_final(True) # end of streaming return + else: + cmd_result.set_is_final(False) # keep streaming return + logger.info("call_chat cmd return_result {}".format(cmd_result.to_json())) + rte.return_result(cmd_result, cmd) + + messages_str = cmd.get_property_string("messages") + messages = json.loads(messages_str) + # messages = [{"role": "user", "content": messages_str}] + stream = False + try: + stream = cmd.get_property_bool("stream") + except Exception as e: + logger.warning("stream property not found, default to False") + + if stream: + self.stream_chat(ts, messages, callback) else: - logger.info("Failed to get response %s", response) + total = self.stream_chat(ts, messages, None) + callback(total, True) # callback once until full answer returned + + def stream_chat(self, ts: datetime.time, messages: List[Any], callback): + logger.info("before stream_chat call {} {}".format(messages, ts)) - def call_with_stream( - self, rte: RteEnv, ts: datetime.time, inputText: str, messages: List[Any] - ): if self.need_interrupt(ts): - logger.warning("out of date, %s, %s", self.outdateTs, ts) + logger.warning("out of date, %s, %s", self.outdate_ts, ts) return - if len(self.ongoing) > 0: - messages.append({"role": "assistant", "content": self.ongoing}) - messages.append({"role": "user", "content": inputText}) - logger.info("before call %s %s", messages, ts) responses = dashscope.Generation.call( self.model, @@ -115,54 +139,47 @@ def call_with_stream( stream=True, # set streaming output incremental_output=True, # get streaming output incrementally ) + total = "" partial = "" for response in responses: if self.need_interrupt(ts): - if len(self.ongoing) > 0: - self.on_msg("user", inputText) - self.on_msg("assistant", self.ongoing) - self.ongoing = "" - logger.warning("out of date, %s, %s", self.outdateTs, ts) - return + logger.warning("out of date, %s, %s", self.outdate_ts, ts) + total += partial + partial = "" # discard not sent + break if response.status_code == HTTPStatus.OK: temp = response.output.choices[0]["message"]["content"] if len(temp) == 0: continue partial += temp - self.ongoing += temp if (isEnd(temp) and len(partial) > 10) or len(partial) > 50: - d = Data.create("text_data") - d.set_property_bool("end_of_segment", isEnd(partial)) - d.set_property_string("text", partial) - rte.send_data(d) + if callback is not None: + callback(partial, False) total += partial partial = "" else: - logger.info( - "Request id: %s, Status code: %s, error code: %s, error message: %s" - % ( + logger.warning( + "request_id: {}, status_code: {}, error code: {}, error message: {}".format( response.request_id, response.status_code, response.code, response.message, ) ) - return + break + + if len(total) > 0 or len(partial) > 0: # make sure no empty answer + if callback is not None: + callback(partial, True) if len(partial) > 0: - d = Data.create("text_data") - d.set_property_bool("end_of_segment", True) - d.set_property_string("text", partial) - rte.send_data(d) total += partial partial = "" - self.ongoing = "" - self.on_msg("user", inputText) - self.on_msg("assistant", total) - logger.info("on response %s", total) + logger.info("stream_chat full_answer {}".format(total)) + return total def on_start(self, rte: RteEnv) -> None: - logger.info("QWenLLMExtension on_start") + logger.info("on_start") self.api_key = rte.get_property_string("api_key") self.model = rte.get_property_string("model") self.prompt = rte.get_property_string("prompt") @@ -174,34 +191,35 @@ def on_start(self, rte: RteEnv) -> None: rte.on_start_done() def on_stop(self, rte: RteEnv) -> None: - logger.info("QWenLLMExtension on_stop") + logger.info("on_stop") self.stopped = True - self.queue.put(None) self.flush() - self.thread.join() + self.queue.put(None) + if self.thread is not None: + self.thread.join() + self.thread = None rte.on_stop_done() - + def flush(self): - logger.info("QWenLLMExtension flush") + self.outdate_ts = datetime.now() while not self.queue.empty(): self.queue.get() def on_data(self, rte: RteEnv, data: Data) -> None: - logger.info("QWenLLMExtension on_data") + logger.info("on_data") is_final = data.get_property_bool("is_final") if not is_final: logger.info("ignore non final") return - inputText = data.get_property_string("text") - if len(inputText) == 0: + input_text = data.get_property_string("text") + if len(input_text) == 0: logger.info("ignore empty text") return ts = datetime.now() - - logger.info("on data %s, %s", inputText, ts) - self.queue.put((inputText, ts)) + logger.info("on data %s, %s", input_text, ts) + self.queue.put((input_text, ts)) def async_handle(self, rte: RteEnv): while not self.stopped: @@ -209,31 +227,36 @@ def async_handle(self, rte: RteEnv): value = self.queue.get() if value is None: break - inputText, ts = value + input, ts = value if self.need_interrupt(ts): continue - logger.info("fetch from queue %s", inputText) - history = self.get_messages() - self.call_with_stream(rte, ts, inputText, history) + + if isinstance(input, str): + logger.info("fetched from queue {}".format(input)) + self.complete_with_history(rte, ts, input) + else: + logger.info("fetched from queue {}".format(input.get_name())) + self.call_chat(rte, ts, input) except Exception as e: logger.exception(e) def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: - logger.info("QWenLLMExtension on_cmd") - cmd_json = cmd.to_json() - logger.info("QWenLLMExtension on_cmd json: %s", cmd_json) - - cmdName = cmd.get_name() - if cmdName == "flush": - self.outdateTs = datetime.now() - # self.flush() + ts = datetime.now() + cmd_name = cmd.get_name() + logger.info("on_cmd {}, {}".format(cmd_name, ts)) + + if cmd_name == "flush": + self.flush() cmd_out = Cmd.create("flush") rte.send_cmd( cmd_out, - lambda rte, result: print("QWenLLMExtensionAddon send_cmd done"), + lambda rte, result: logger.info("send_cmd flush done"), ) + elif cmd_name == "call_chat": + self.queue.put((cmd, ts)) + return # cmd_result will be returned once it's processed else: - logger.info("unknown cmd %s", cmdName) + logger.info("unknown cmd {}".format(cmd_name)) cmd_result = CmdResult.create(StatusCode.OK) rte.return_result(cmd_result, cmd) From a90761e1bdd96baefb32c20ed241038935608752 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 8 Aug 2024 00:27:17 -0700 Subject: [PATCH 09/34] feat: basic chat engine --- .../llama_index_chat_engine/__init__.py | 4 + .../llama_index_chat_engine/addon.py | 10 + .../llama_index_chat_engine/astra_llm.py | 148 +++++++++++++ .../llama_index_chat_engine/extension.py | 203 ++++++++++++++++++ .../extension/llama_index_chat_engine/log.py | 13 ++ .../llama_index_chat_engine/manifest.json | 97 +++++++++ .../llama_index_chat_engine/property.json | 1 + .../llama_index_chat_engine/requirements.txt | 1 + 8 files changed, 477 insertions(+) create mode 100644 agents/addon/extension/llama_index_chat_engine/__init__.py create mode 100644 agents/addon/extension/llama_index_chat_engine/addon.py create mode 100644 agents/addon/extension/llama_index_chat_engine/astra_llm.py create mode 100644 agents/addon/extension/llama_index_chat_engine/extension.py create mode 100644 agents/addon/extension/llama_index_chat_engine/log.py create mode 100644 agents/addon/extension/llama_index_chat_engine/manifest.json create mode 100644 agents/addon/extension/llama_index_chat_engine/property.json create mode 100644 agents/addon/extension/llama_index_chat_engine/requirements.txt diff --git a/agents/addon/extension/llama_index_chat_engine/__init__.py b/agents/addon/extension/llama_index_chat_engine/__init__.py new file mode 100644 index 00000000..55408286 --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/__init__.py @@ -0,0 +1,4 @@ +from . import addon +from .log import logger + +logger.info("llama_index_chat_engine extension loaded") diff --git a/agents/addon/extension/llama_index_chat_engine/addon.py b/agents/addon/extension/llama_index_chat_engine/addon.py new file mode 100644 index 00000000..03586528 --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/addon.py @@ -0,0 +1,10 @@ +from rte import Addon, register_addon_as_extension, RteEnv +from .extension import LlamaIndexExtension +from .log import logger + + +@register_addon_as_extension("llama_index_chat_engine") +class LlamaIndexExtensionAddon(Addon): + def on_create_instance(self, rte: RteEnv, addon_name: str, context) -> None: + logger.info("on_create_instance") + rte.on_create_instance_done(LlamaIndexExtension(addon_name), context) diff --git a/agents/addon/extension/llama_index_chat_engine/astra_llm.py b/agents/addon/extension/llama_index_chat_engine/astra_llm.py new file mode 100644 index 00000000..a642603c --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/astra_llm.py @@ -0,0 +1,148 @@ +from typing import Any, Sequence +import json, queue +import threading + +from llama_index.core.base.llms.types import ( + LLMMetadata, + MessageRole, + ChatMessage, + ChatResponse, + CompletionResponse, + ChatResponseGen, + CompletionResponseGen, +) + +from llama_index.core.llms.callbacks import llm_chat_callback, llm_completion_callback + +from llama_index.core.llms.custom import CustomLLM +from .log import logger +from rte import Cmd, StatusCode, CmdResult, RteEnv + + +def chat_from_astra_response(cmd_result: CmdResult) -> ChatResponse: + status = cmd_result.get_status_code() + if status != StatusCode.OK: + return None + text_data = cmd_result.get_property_string("text") + return ChatResponse(message=ChatMessage(content=text_data)) + + +def _messages_str_from_chat_messages(messages: Sequence[ChatMessage]) -> str: + messages_list = [] + for message in messages: + messages_list.append( + {"role": message.role, "content": "{}".format(message.content)} + ) + return json.dumps(messages_list, ensure_ascii=False) + + +class ASTRALLM(CustomLLM): + rte: Any + + def __init__(self, rte): + """Creates a new ASTRA model interface.""" + super().__init__() + self.rte = rte + + @property + def metadata(self) -> LLMMetadata: + return LLMMetadata( + # TODO: fix metadata + context_window=1024, + num_output=512, + model_name="astra_llm", + is_chat_model=True, + ) + + @llm_chat_callback() + def chat(self, messages: Sequence[ChatMessage], **kwargs: Any) -> ChatResponse: + logger.debug("ASTRALLM chat start") + + resp: ChatResponse + wait_event = threading.Event() + + def callback(_, result): + logger.debug("ASTRALLM chat callback done") + nonlocal resp + nonlocal wait_event + resp = chat_from_astra_response(result) + wait_event.set() + + messages_str = _messages_str_from_chat_messages(messages) + + cmd = Cmd.create("call_chat") + cmd.set_property_string("messages", messages_str) + cmd.set_property_bool("stream", False) + logger.info( + "ASTRALLM chat send_cmd {}, messages {}".format( + cmd.get_name(), messages_str + ) + ) + + self.rte.send_cmd(cmd, callback) + wait_event.wait() + return resp + + @llm_completion_callback() + def complete( + self, prompt: str, formatted: bool = False, **kwargs: Any + ) -> CompletionResponse: + logger.warning("ASTRALLM complete hasn't been implemented yet") + + @llm_chat_callback() + def stream_chat( + self, messages: Sequence[ChatMessage], **kwargs: Any + ) -> ChatResponseGen: + logger.debug("ASTRALLM stream_chat start") + + cur_tokens = "" + resp_queue = queue.Queue() + + def gen() -> ChatResponseGen: + while True: + delta_text = resp_queue.get() + if delta_text is None: + break + + yield ChatResponse( + message=ChatMessage(content=delta_text, role=MessageRole.ASSISTANT), + delta=delta_text, + ) + + def callback(_, result): + nonlocal cur_tokens + nonlocal resp_queue + + status = result.get_status_code() + if status != StatusCode.OK: + logger.warn("ASTRALLM stream_chat callback status {}".format(status)) + resp_queue.put(None) + return + + cur_tokens = result.get_property_string("text") + logger.debug("ASTRALLM stream_chat callback text [{}]".format(cur_tokens)) + resp_queue.put(cur_tokens) + if result.get_is_final(): + resp_queue.put(None) + + messages_str = _messages_str_from_chat_messages(messages) + + cmd = Cmd.create("call_chat") + cmd.set_property_string("messages", messages_str) + cmd.set_property_bool("stream", True) + logger.info( + "ASTRALLM stream_chat send_cmd {}, messages {}".format( + cmd.get_name(), messages_str + ) + ) + self.rte.send_cmd(cmd, callback) + return gen() + + def stream_complete( + self, prompt: str, formatted: bool = False, **kwargs: Any + ) -> CompletionResponseGen: + logger.warning("ASTRALLM stream_complete hasn't been implemented yet") + + @classmethod + def class_name(cls) -> str: + return "astra_llm" diff --git a/agents/addon/extension/llama_index_chat_engine/extension.py b/agents/addon/extension/llama_index_chat_engine/extension.py new file mode 100644 index 00000000..2c189dc0 --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/extension.py @@ -0,0 +1,203 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-05. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from rte import ( + Extension, + RteEnv, + Cmd, + Data, + StatusCode, + CmdResult, +) +from .log import logger +from .astra_llm import ASTRALLM +from .astra_retriever import ASTRARetriever +import queue, threading +from datetime import datetime +from llama_index.core.chat_engine import SimpleChatEngine, ContextChatEngine +from llama_index.core.storage.chat_store import SimpleChatStore +from llama_index.core.memory import ChatMemoryBuffer + +PROPERTY_CHAT_MEMORY_TOKEN_LIMIT = "chat_memory_token_limit" +PROPERTY_GREETING = "greeting" + + +class LlamaIndexExtension(Extension): + def __init__(self, name: str): + super().__init__(name) + self.queue = queue.Queue() + self.thread = None + self.stop = False + + self.collection_name = "" + self.outdate_ts = datetime.now() + self.chat_memory_token_limit = 3000 + self.chat_memory = None + + def _send_text_data(self, rte: RteEnv, text: str, end_of_segment: bool): + try: + output_data = Data.create("text_data") + output_data.set_property_string("text", text) + output_data.set_property_bool("end_of_segment", end_of_segment) + rte.send_data(output_data) + logger.info("text [{}] end_of_segment {} sent".format(text, end_of_segment)) + except Exception as err: + logger.info( + "text [{}] end_of_segment {} send failed, err {}".format( + text, end_of_segment, err + ) + ) + + def on_start(self, rte: RteEnv) -> None: + logger.info("on_start") + + greeting = None + try: + greeting = rte.get_property_string(PROPERTY_GREETING) + except Exception as err: + logger.warning(f"get {PROPERTY_GREETING} property failed, err: {err}") + + try: + self.chat_memory_token_limit = rte.get_property_int( + PROPERTY_CHAT_MEMORY_TOKEN_LIMIT + ) + except Exception as err: + logger.warning( + f"get {PROPERTY_CHAT_MEMORY_TOKEN_LIMIT} property failed, err: {err}" + ) + + self.thread = threading.Thread(target=self.async_handle, args=[rte]) + self.thread.start() + + # enable chat memory + self.chat_memory = ChatMemoryBuffer.from_defaults( + token_limit=self.chat_memory_token_limit, + chat_store=SimpleChatStore(), + ) + + # Send greeting if available + if greeting is not None: + self._send_text_data(rte, greeting, True) + + rte.on_start_done() + + def on_stop(self, rte: RteEnv) -> None: + logger.info("on_stop") + + self.stop = True + self.flush() + self.queue.put(None) + if self.thread is not None: + self.thread.join() + self.thread = None + self.chat_memory = None + + rte.on_stop_done() + + def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: + + cmd_name = cmd.get_name() + logger.info("on_cmd {}".format(cmd_name)) + if cmd_name == "file_chunked": + coll = cmd.get_property_string("collection_name") + self.collection_name = coll + + file_chunked_text = "Your document has been processed. Please wait a moment while we process your document for you. " + self._send_text_data(rte, file_chunked_text, True) + elif cmd_name == "file_downloaded": + file_downloaded_text = "Your document has been received. Please wait a moment while we process it for you. " + self._send_text_data(rte, file_downloaded_text, True) + elif cmd_name == "flush": + self.flush() + rte.send_cmd(Cmd.create("flush"), None) + + cmd_result = CmdResult.create(StatusCode.OK) + cmd_result.set_property_string("detail", "ok") + rte.return_result(cmd_result, cmd) + + def on_data(self, rte: RteEnv, data: Data) -> None: + is_final = data.get_property_bool("is_final") + if not is_final: + logger.info("on_data ignore non final") + return + + inputText = data.get_property_string("text") + if len(inputText) == 0: + logger.info("on_data ignore empty text") + return + + ts = datetime.now() + + logger.info("on_data text [%s], ts [%s]", inputText, ts) + self.queue.put((inputText, ts)) + + def async_handle(self, rte: RteEnv): + logger.info("async_handle started") + while not self.stop: + try: + value = self.queue.get() + if value is None: + break + input_text, ts = value + if ts < self.outdate_ts: + logger.info( + "text [%s] ts [%s] dropped due to outdated", input_text, ts + ) + continue + logger.info("process input text [%s] ts [%s]", input_text, ts) + + # prepare chat engine + chat_engine = None + if len(self.collection_name) > 0: + chat_engine = ContextChatEngine.from_defaults( + llm=ASTRALLM(rte=rte), + retriever=ASTRARetriever(rte=rte, coll=self.collection_name), + memory=self.chat_memory, + system_prompt=( + "You are an expert Q&A system that is trusted around the world.\n" + "Always answer the query using the provided context information, " + "and not prior knowledge.\n" + "Some rules to follow:\n" + "1. Never directly reference the given context in your answer.\n" + "2. Avoid statements like 'Based on the context, ...' or " + "'The context information ...' or anything along " + "those lines." + ), + ) + else: + chat_engine = SimpleChatEngine.from_defaults( + llm=ASTRALLM(rte=rte), + system_prompt="You are an expert Q&A system that is trusted around the world.\n", + memory=self.chat_memory, + ) + + resp = chat_engine.stream_chat(input_text) + for cur_token in resp.response_gen: + if self.stop: + break + if ts < self.outdate_ts: + logger.info( + "stream_chat coming responses dropped due to outdated for input text [%s] ts [%s] ", + input_text, + ts, + ) + break + text = str(cur_token) + + # send out + self._send_text_data(rte, text, False) + + # send out end_of_segment + self._send_text_data(rte, "", True) + except Exception as e: + logger.exception(e) + logger.info("async_handle stoped") + + def flush(self): + self.outdate_ts = datetime.now() + while not self.queue.empty(): + self.queue.get() diff --git a/agents/addon/extension/llama_index_chat_engine/log.py b/agents/addon/extension/llama_index_chat_engine/log.py new file mode 100644 index 00000000..0804a279 --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/log.py @@ -0,0 +1,13 @@ +import logging + +logger = logging.getLogger("llama_index_chat_engine") +logger.setLevel(logging.INFO) + +formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - [%(filename)s:%(lineno)d] - %(message)s" +) + +console_handler = logging.StreamHandler() +console_handler.setFormatter(formatter) + +logger.addHandler(console_handler) diff --git a/agents/addon/extension/llama_index_chat_engine/manifest.json b/agents/addon/extension/llama_index_chat_engine/manifest.json new file mode 100644 index 00000000..2910ce5f --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/manifest.json @@ -0,0 +1,97 @@ +{ + "type": "extension", + "name": "llama_index_chat_engine", + "version": "0.4.0", + "language": "python", + "dependencies": [ + { + "type": "system", + "name": "rte_runtime_python", + "version": "0.4" + } + ], + "api": { + "property": { + "chat_memory_token_limit": { + "type": "int32" + }, + "greeting": { + "type": "string" + } + }, + "data_in": [ + { + "name": "text_data", + "property": { + "text": { + "type": "string" + }, + "is_final": { + "type": "bool" + } + } + } + ], + "data_out": [ + { + "name": "text_data", + "property": { + "text": { + "type": "string" + }, + "end_of_segment": { + "type": "bool" + } + } + } + ], + "cmd_in": [ + { + "name": "flush" + }, + { + "name": "file_downloaded" + }, + { + "name": "file_chunked", + "property": { + "collection_name": { + "type": "string" + } + }, + "required": [ + "collection_name" + ] + } + ], + "cmd_out": [ + { + "name": "flush" + }, + { + "name": "call_chat", + "property": { + "messages": { + "type": "string" + }, + "stream": { + "type": "bool" + } + }, + "required": [ + "messages" + ], + "result": { + "property": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ] + } + } + ] + } +} \ No newline at end of file diff --git a/agents/addon/extension/llama_index_chat_engine/property.json b/agents/addon/extension/llama_index_chat_engine/property.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/agents/addon/extension/llama_index_chat_engine/requirements.txt b/agents/addon/extension/llama_index_chat_engine/requirements.txt new file mode 100644 index 00000000..f1987506 --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/requirements.txt @@ -0,0 +1 @@ +llama_index From f18205abb5402fb6060d6beaef304a6eeac38402 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 8 Aug 2024 01:02:51 -0700 Subject: [PATCH 10/34] feat: rag querying --- .../manifest.json | 23 +++++ .../embedding_extension.py | 5 +- .../astra_embedding.py | 65 ++++++++++++++ .../astra_retriever.py | 88 +++++++++++++++++++ .../llama_index_chat_engine/manifest.json | 61 +++++++++++++ 5 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 agents/addon/extension/llama_index_chat_engine/astra_embedding.py create mode 100644 agents/addon/extension/llama_index_chat_engine/astra_retriever.py diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/manifest.json b/agents/addon/extension/aliyun_analyticdb_vector_storage/manifest.json index c7a64f46..028a2154 100644 --- a/agents/addon/extension/aliyun_analyticdb_vector_storage/manifest.json +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/manifest.json @@ -67,6 +67,29 @@ "type": "float64" } } + }, + "required": [ + "collection_name", + "top_k", + "embedding" + ], + "result": { + "property": { + "response": { + "type": "array", + "items": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "score": { + "type": "float64" + } + } + } + } + } } }, { diff --git a/agents/addon/extension/aliyun_text_embedding/embedding_extension.py b/agents/addon/extension/aliyun_text_embedding/embedding_extension.py index f3556c4c..284a7918 100644 --- a/agents/addon/extension/aliyun_text_embedding/embedding_extension.py +++ b/agents/addon/extension/aliyun_text_embedding/embedding_extension.py @@ -89,7 +89,7 @@ def call_with_str(self, message: str) -> CmdResult: if response.status_code == HTTPStatus.OK: cmd_result = CmdResult.create(StatusCode.OK) - cmd_result.set_property_from_json(FIELD_KEY_EMBEDDING, response.output["embeddings"][0]["embedding"]) + cmd_result.set_property_from_json(FIELD_KEY_EMBEDDING, json.dumps(response.output["embeddings"][0]["embedding"])) return cmd_result else: cmd_result = CmdResult.create(StatusCode.ERROR) @@ -124,6 +124,9 @@ def call_with_strs(self, messages: List[str]) -> CmdResult: logger.info("embedding call finished for inputs len {}, batch_counter {}, results len {}, cost {}ms ".format(len(messages), batch_counter, len(result["embeddings"]), int((datetime.now() - start_time).total_seconds() * 1000))) if result is not None: cmd_result = CmdResult.create(StatusCode.OK) + + # TODO: too slow `set_property_to_json`, so use `set_property_string` at the moment as workaround + # will be replaced once `set_property_to_json` improved cmd_result.set_property_string(FIELD_KEY_EMBEDDINGS, json.dumps(result["embeddings"])) return cmd_result else: diff --git a/agents/addon/extension/llama_index_chat_engine/astra_embedding.py b/agents/addon/extension/llama_index_chat_engine/astra_embedding.py new file mode 100644 index 00000000..fb96da77 --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/astra_embedding.py @@ -0,0 +1,65 @@ +from typing import Any, List +import threading +from llama_index.core.embeddings import BaseEmbedding +from .log import logger +import json +from rte import ( + Cmd, + CmdResult, +) + +EMBED_CMD = "embed" + + +def embed_from_resp(cmd_result: CmdResult) -> List[float]: + embedding_output_json = cmd_result.get_property_to_json("embedding") + return json.loads(embedding_output_json) + + +class ASTRAEmbedding(BaseEmbedding): + rte: Any + + def __init__(self, rte): + """Creates a new ASTRA embedding interface.""" + super().__init__() + self.rte = rte + + @classmethod + def class_name(cls) -> str: + return "astra_embedding" + + async def _aget_query_embedding(self, query: str) -> List[float]: + return self._get_query_embedding(query) + + async def _aget_text_embedding(self, text: str) -> List[float]: + return self._get_text_embedding(text) + + def _get_query_embedding(self, query: str) -> List[float]: + logger.info( + "ASTRAEmbedding generate embeddings for the query: {}".format(query) + ) + wait_event = threading.Event() + resp: List[float] + + def callback(_, result): + nonlocal resp + nonlocal wait_event + + logger.debug("ASTRAEmbedding embedding received") + resp = embed_from_resp(result) + wait_event.set() + + cmd_out = Cmd.create(EMBED_CMD) + cmd_out.set_property_string("input", query) + + self.rte.send_cmd(cmd_out, callback) + wait_event.wait() + return resp + + def _get_text_embedding(self, text: str) -> List[float]: + return self._get_query_embedding(text) + + # for texts embedding, will not be called in this module + def _get_text_embeddings(self, texts: List[str]) -> List[List[float]]: + logger.warning("not implemented") + return [] diff --git a/agents/addon/extension/llama_index_chat_engine/astra_retriever.py b/agents/addon/extension/llama_index_chat_engine/astra_retriever.py new file mode 100644 index 00000000..c74dfb99 --- /dev/null +++ b/agents/addon/extension/llama_index_chat_engine/astra_retriever.py @@ -0,0 +1,88 @@ +import time, json, threading +from typing import Any, List +from llama_index.core.schema import QueryBundle, TextNode +from llama_index.core.schema import NodeWithScore +from llama_index.core.retrievers import BaseRetriever + +from .log import logger +from .astra_embedding import ASTRAEmbedding +from rte import ( + RteEnv, + Cmd, + StatusCode, + CmdResult, +) + + +def format_node_result(cmd_result: CmdResult) -> List[NodeWithScore]: + logger.info("ASTRARetriever retrieve response {}".format(cmd_result.to_json())) + status = cmd_result.get_status_code() + try: + contents_json = cmd_result.get_property_to_json("response") + except Exception as e: + logger.warning(f"Failed to get response from cmd_result: {e}") + return [ + NodeWithScore( + node=TextNode(), + score=0.0, + ) + ] + contents = json.loads(contents_json) + if status != StatusCode.OK or len(contents) == 0: + return [ + NodeWithScore( + node=TextNode(), + score=0.0, + ) + ] + + nodes = [] + for result in contents: + text_node = TextNode( + text=result["content"], + ) + nodes.append(NodeWithScore(node=text_node, score=result["score"])) + return nodes + + +class ASTRARetriever(BaseRetriever): + rte: Any + embed_model: ASTRAEmbedding + + def __init__(self, rte: RteEnv, coll: str): + super().__init__() + try: + self.rte = rte + self.embed_model = ASTRAEmbedding(rte=rte) + self.collection_name = coll + except Exception as e: + logger.error(f"Failed to initialize ASTRARetriever: {e}") + + def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]: + logger.info("ASTRARetriever retrieve: {}".format(query_bundle.to_json)) + + wait_event = threading.Event() + resp: List[NodeWithScore] = [] + + def cmd_callback(_, result): + nonlocal resp + nonlocal wait_event + resp = format_node_result(result) + wait_event.set() + logger.debug("ASTRARetriever callback done") + + embedding = self.embed_model.get_query_embedding(query=query_bundle.query_str) + + query_cmd = Cmd.create("query_vector") + query_cmd.set_property_string("collection_name", self.collection_name) + query_cmd.set_property_int("top_k", 3) # TODO: configable + query_cmd.set_property_from_json("embedding", json.dumps(embedding)) + logger.info( + "ASTRARetriever send_cmd, collection_name: {}, embedding len: {}".format( + self.collection_name, len(embedding) + ) + ) + self.rte.send_cmd(query_cmd, cmd_callback) + + wait_event.wait() + return resp diff --git a/agents/addon/extension/llama_index_chat_engine/manifest.json b/agents/addon/extension/llama_index_chat_engine/manifest.json index 2910ce5f..11ca8973 100644 --- a/agents/addon/extension/llama_index_chat_engine/manifest.json +++ b/agents/addon/extension/llama_index_chat_engine/manifest.json @@ -91,6 +91,67 @@ "text" ] } + }, + { + "name": "embed", + "property": { + "input": { + "type": "string" + } + }, + "required": [ + "input" + ], + "result": { + "property": { + "embedding": { + "type": "array", + "items": { + "type": "float64" + } + } + } + } + }, + { + "name": "query_vector", + "property": { + "collection_name": { + "type": "string" + }, + "top_k": { + "type": "int64" + }, + "embedding": { + "type": "array", + "items": { + "type": "float64" + } + } + }, + "required": [ + "collection_name", + "top_k", + "embedding" + ], + "result": { + "property": { + "response": { + "type": "array", + "items": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "score": { + "type": "float64" + } + } + } + } + } + } } ] } From 04d1a3fba5811ea072d14f042da3686a19e78010 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 8 Aug 2024 01:12:13 -0700 Subject: [PATCH 11/34] feat: add qwen.rag graph --- agents/property.json.example | 348 +++++++++++++++++++++++++++++++++++ 1 file changed, 348 insertions(+) diff --git a/agents/property.json.example b/agents/property.json.example index c84ad926..c63abb40 100644 --- a/agents/property.json.example +++ b/agents/property.json.example @@ -1387,6 +1387,354 @@ ] } ] + }, + { + "name": "va.qwen.rag", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "extension_group": "rtc", + "addon": "agora_rtc", + "name": "agora_rtc", + "property": { + "app_id": "", + "token": "", + "channel": "astra_agents_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "", + "agora_asr_vendor_region": "", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "extension_group": "llm", + "addon": "qwen_llm_python", + "name": "qwen_llm", + "property": { + "api_key": "", + "model": "qwen-max", + "max_tokens": 512, + "prompt": "", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "extension_group": "tts", + "addon": "cosy_tts", + "name": "cosy_tts", + "property": { + "api_key": "", + "model": "cosyvoice-v1", + "voice": "longxiaochun", + "sample_rate": 16000 + } + }, + { + "type": "extension", + "extension_group": "chat_transcriber", + "addon": "chat_transcriber_python", + "name": "chat_transcriber" + }, + { + "type": "extension", + "extension_group": "interrupt_detector", + "addon": "interrupt_detector_python", + "name": "interrupt_detector" + }, + { + "type": "extension", + "extension_group": "http_server", + "addon": "http_server_python", + "name": "http_server", + "property": { + "listen_addr": "127.0.0.1", + "listen_port": 8080 + } + }, + { + "type": "extension", + "extension_group": "embedding", + "addon": "aliyun_text_embedding", + "name": "embedding", + "property": { + "api_key": "", + "model": "text-embedding-v3" + } + }, + { + "type": "extension", + "extension_group": "vector_storage", + "addon": "aliyun_analyticdb_vector_storage", + "name": "vector_storage", + "property": { + "alibaba_cloud_access_key_id": "", + "alibaba_cloud_access_key_secret": "", + "adbpg_instance_id": "", + "adbpg_instance_region": "cn-shanghai", + "adbpg_account": "", + "adbpg_account_password": "", + "adbpg_namespace": "", + "adbpg_namespace_password": "" + } + }, + { + "type": "extension", + "extension_group": "file_chunker", + "addon": "file_chunker", + "name": "file_chunker", + "property": {} + }, + { + "type": "extension", + "extension_group": "llama_index", + "addon": "llama_index_chat_engine", + "name": "llama_index", + "property": { + "greeting": "ASTRA agent connected. How can i help you today?", + "chat_memory_token_limit": 3000 + } + } + ], + "connections": [ + { + "extension_group": "rtc", + "extension": "agora_rtc", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector" + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber" + } + ] + } + ] + }, + { + "extension_group": "interrupt_detector", + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ] + }, + { + "extension_group": "llama_index", + "extension": "llama_index", + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "tts", + "extension": "cosy_tts" + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber", + "cmd_conversions": [ + { + "cmd": { + "type": "per_property", + "keep_original": true, + "rules": [ + { + "path": "is_final", + "type": "fixed_value", + "value": "bool(true)" + }, + { + "path": "stream_id", + "type": "fixed_value", + "value": "uint32(999)" + } + ] + } + } + ] + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + }, + { + "extension_group": "tts", + "extension": "cosy_tts" + } + ] + }, + { + "name": "call_chat", + "dest": [ + { + "extension_group": "llm", + "extension": "qwen_llm" + } + ] + }, + { + "name": "embed", + "dest": [ + { + "extension_group": "embedding", + "extension": "embedding" + } + ] + }, + { + "name": "query_vector", + "dest": [ + { + "extension_group": "vector_storage", + "extension": "vector_storage" + } + ] + } + ] + }, + { + "extension_group": "tts", + "extension": "cosy_tts", + "pcm_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ], + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "chat_transcriber", + "extension": "chat_transcriber", + "data": [ + { + "name": "data", + "dest": [ + { + "extension_group": "rtc", + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension_group": "http_server", + "extension": "http_server", + "cmd": [ + { + "name": "file_downloaded", + "dest": [ + { + "extension_group": "file_chunker", + "extension": "file_chunker" + }, + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ] + }, + { + "extension_group": "file_chunker", + "extension": "file_chunker", + "cmd": [ + { + "name": "embed_batch", + "dest": [ + { + "extension_group": "embedding", + "extension": "embedding" + } + ] + }, + { + "name": "create_collection", + "dest": [ + { + "extension_group": "vector_storage", + "extension": "vector_storage" + } + ] + }, + { + "name": "upsert_vector", + "dest": [ + { + "extension_group": "vector_storage", + "extension": "vector_storage" + } + ] + }, + { + "name": "file_chunked", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + } + ] + } + ] } ] } From 34ccc294e4fdf65ac74839ebd77c200d96999a96 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Thu, 8 Aug 2024 02:32:06 -0700 Subject: [PATCH 12/34] fix: cozy tts drain and interrupt --- .../extension/cosy_tts/cosy_tts_extension.py | 136 ++++++++++-------- 1 file changed, 75 insertions(+), 61 deletions(-) diff --git a/agents/addon/extension/cosy_tts/cosy_tts_extension.py b/agents/addon/extension/cosy_tts/cosy_tts_extension.py index 368d10ea..cd57d406 100644 --- a/agents/addon/extension/cosy_tts/cosy_tts_extension.py +++ b/agents/addon/extension/cosy_tts/cosy_tts_extension.py @@ -15,7 +15,6 @@ Data, StatusCode, CmdResult, - MetadataInfo, ) from typing import List, Any import dashscope @@ -27,17 +26,21 @@ class CosyTTSCallback(ResultCallback): - _player = None - _stream = None - - def __init__(self, rte: RteEnv, sample_rate: int): + def __init__(self, rte: RteEnv, sample_rate: int, need_interrupt_callback): super().__init__() self.rte = rte self.sample_rate = sample_rate self.frame_size = int(self.sample_rate * 1 * 2 / 100) - self.canceled = False + self.ts = datetime.now() + self.need_interrupt_callback = need_interrupt_callback self.closed = False + def need_interrupt(self) -> bool: + return self.need_interrupt_callback(self.ts) + + def set_input_ts(self, ts: datetime): + self.ts = ts + def on_open(self): logger.info("websocket is open.") @@ -71,11 +74,8 @@ def get_frame(self, data: bytes) -> PcmFrame: f.unlock_buf(buff) return f - def cancel(self) -> None: - self.canceled = True - def on_data(self, data: bytes) -> None: - if self.canceled: + if self.need_interrupt(): return # logger.info("audio result length: %d, %d", len(data), self.frame_size) @@ -83,13 +83,13 @@ def on_data(self, data: bytes) -> None: chunk = int(len(data) / self.frame_size) offset = 0 for i in range(0, chunk): - if self.canceled: + if self.need_interrupt(): return f = self.get_frame(data[offset : offset + self.frame_size]) self.rte.send_pcm_frame(f) offset += self.frame_size - if self.canceled: + if self.need_interrupt(): return if offset < len(data): size = len(data) - offset @@ -109,15 +109,15 @@ def __init__(self, name: str): self.tts = None self.callback = None self.format = None - self.outdateTs = datetime.now() + + self.outdate_ts = datetime.now() self.stopped = False self.thread = None self.queue = queue.Queue() - self.mutex = threading.Lock() def on_start(self, rte: RteEnv) -> None: - logger.info("CosyTTSExtension on_start") + logger.info("on_start") self.api_key = rte.get_property_string("api_key") self.voice = rte.get_property_string("voice") self.model = rte.get_property_string("model") @@ -138,7 +138,7 @@ def on_start(self, rte: RteEnv) -> None: elif self.sample_rate == 48000: f = AudioFormat.PCM_48000HZ_MONO_16BIT else: - logger.info("unknown sample rate %d", self.sample_rate) + logger.error("unknown sample rate %d", self.sample_rate) exit() self.format = f @@ -148,16 +148,18 @@ def on_start(self, rte: RteEnv) -> None: rte.on_start_done() def on_stop(self, rte: RteEnv) -> None: - logger.info("CosyTTSExtension on_stop") + logger.info("on_stop") self.stopped = True - self.queue.put(None) self.flush() - self.thread.join() + self.queue.put(None) + if self.thread is not None: + self.thread.join() + self.thread = None rte.on_stop_done() def need_interrupt(self, ts: datetime.time) -> bool: - return self.outdateTs > ts and (self.outdateTs - ts).total_seconds() > 1 + return self.outdate_ts > ts def async_handle(self, rte: RteEnv): try: @@ -168,29 +170,33 @@ def async_handle(self, rte: RteEnv): value = self.queue.get() if value is None: break - inputText, ts = value - if len(inputText) == 0: - logger.warning("empty input for interrupt") - if tts is not None: - try: - tts.streaming_cancel() - except Exception as e: - logger.exception(e) - if callback is not None: - callback.cancel() + input_text, ts, end_of_segment = value + + # clear tts if old one is closed already + if callback is not None and callback.closed is True: + tts = None + callback = None + + # cancel last streaming call to avoid unprocessed audio coming back + if ( + callback is not None + and tts is not None + and callback.need_interrupt() + ): + tts.streaming_cancel() tts = None callback = None - continue if self.need_interrupt(ts): + logger.info("drop outdated input") continue - if callback is not None and callback.closed is True: - tts = None - - if tts is None: + # create new tts if needed + if tts is None or callback is None: logger.info("creating tts") - callback = CosyTTSCallback(rte, self.sample_rate) + callback = CosyTTSCallback( + rte, self.sample_rate, self.need_interrupt + ) tts = SpeechSynthesizer( model=self.model, voice=self.voice, @@ -198,49 +204,57 @@ def async_handle(self, rte: RteEnv): callback=callback, ) - logger.info("on message [%s]", inputText) - tts.streaming_call(inputText) - tts.streaming_complete() + logger.info( + "on message [{}] ts [{}] end_of_segment [{}]".format( + input_text, ts, end_of_segment + ) + ) + + # make sure new data won't be marked as outdated + callback.set_input_ts(ts) + + if len(input_text) > 0: + # last segment may have empty text but is_end is true + tts.streaming_call(input_text) + + # complete the streaming call to drain remained audio if end_of_segment is true + if end_of_segment: + try: + tts.streaming_complete() + except Exception as e: + logger.warning(e) + tts = None + callback = None except Exception as e: logger.exception(e) logger.exception(traceback.format_exc()) finally: if tts is not None: - tts.streaming_complete() + tts.streaming_cancel() + tts = None + callback = None def flush(self): - logger.info("CosyTTSExtension flush") while not self.queue.empty(): self.queue.get() - self.queue.put(("", datetime.now())) def on_data(self, rte: RteEnv, data: Data) -> None: - logger.info("CosyTTSExtension on_data") inputText = data.get_property_string("text") - if len(inputText) == 0: - logger.info("ignore empty text") - return - - is_end = data.get_property_bool("end_of_segment") + end_of_segment = data.get_property_bool("end_of_segment") - logger.info("on data %s %d", inputText, is_end) - self.queue.put((inputText, datetime.now())) + logger.info("on data {} {}".format(inputText, end_of_segment)) + self.queue.put((inputText, datetime.now(), end_of_segment)) def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: - logger.info("CosyTTSExtension on_cmd") - cmd_json = cmd.to_json() - logger.info("CosyTTSExtension on_cmd json: %s" + cmd_json) - - cmdName = cmd.get_name() - if cmdName == "flush": - self.outdateTs = datetime.now() + cmd_name = cmd.get_name() + logger.info("on_cmd {}".format(cmd_name)) + if cmd_name == "flush": + self.outdate_ts = datetime.now() self.flush() cmd_out = Cmd.create("flush") - rte.send_cmd( - cmd_out, lambda rte, result: print("DefaultExtension send_cmd done") - ) + rte.send_cmd(cmd_out, lambda rte, result: print("send_cmd flush done")) else: - logger.info("unknown cmd %s", cmdName) + logger.info("unknown cmd {}".format(cmd_name)) cmd_result = CmdResult.create(StatusCode.OK) cmd_result.set_property_string("detail", "success") From bd29cfa46ad86c8a7aeb1bd3d4d46be5c1d922d7 Mon Sep 17 00:00:00 2001 From: sunshinexcode <24xinhui@163.com> Date: Fri, 9 Aug 2024 16:57:22 +0800 Subject: [PATCH 13/34] chore(): add api upload --- server/go.mod | 9 ++-- server/go.sum | 57 ++++++++++++++++++++++---- server/internal/code.go | 3 ++ server/internal/config.go | 6 ++- server/internal/http_client.go | 15 +++++++ server/internal/http_server.go | 75 ++++++++++++++++++++++++++++++---- server/internal/worker.go | 59 +++++++++++++++++++++++++- 7 files changed, 202 insertions(+), 22 deletions(-) create mode 100644 server/internal/http_client.go diff --git a/server/go.mod b/server/go.mod index c2324acc..256c8c58 100644 --- a/server/go.mod +++ b/server/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/AgoraIO/Tools/DynamicKey/AgoraDynamicKey/go/src v0.0.0-20240531043742-11bdd9531d08 github.com/gin-gonic/gin v1.9.1 + github.com/go-resty/resty/v2 v2.13.1 github.com/gogf/gf v1.16.9 github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.5.1 @@ -40,10 +41,10 @@ require ( go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/server/go.sum b/server/go.sum index 5be9e17a..ba727db0 100644 --- a/server/go.sum +++ b/server/go.sum @@ -38,6 +38,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= +github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -112,6 +114,7 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= @@ -121,25 +124,63 @@ go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+go golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= diff --git a/server/internal/code.go b/server/internal/code.go index dfb6f6ca..1a583ee0 100644 --- a/server/internal/code.go +++ b/server/internal/code.go @@ -15,10 +15,13 @@ var ( codeErrChannelExisted = NewCode("10003", "channel existed") codeErrChannelEmpty = NewCode("10004", "channel empty") codeErrGenerateTokenFailed = NewCode("10005", "generate token failed") + codeErrSaveFileFailed = NewCode("10006", "save file failed") codeErrProcessPropertyFailed = NewCode("10100", "process property json failed") codeErrStartWorkerFailed = NewCode("10101", "start worker failed") codeErrStopWorkerFailed = NewCode("10102", "stop worker failed") + codeErrHttpStatusNotOk = NewCode("10103", "http status not 200") + codeErrUpdateWorkerFailed = NewCode("10104", "update worker failed") ) func NewCode(code string, msg string) *Code { diff --git a/server/internal/config.go b/server/internal/config.go index a0480166..9297e300 100644 --- a/server/internal/config.go +++ b/server/internal/config.go @@ -10,10 +10,11 @@ type Prop struct { const ( // Extension name extensionNameAgoraRTC = "agora_rtc" - extensionNameBedrockLLM = "bedrock_llm" extensionNameAzureTTS = "azure_tts" + extensionNameBedrockLLM = "bedrock_llm" extensionNameCosyTTS = "cosy_tts" extensionNameElevenlabsTTS = "elevenlabs_tts" + extensionNameHttpServer = "http_server" extensionNameLiteLLM = "litellm" extensionNameOpenaiChatgpt = "openai_chatgpt" extensionNamePollyTTS = "polly_tts" @@ -116,6 +117,9 @@ var ( {ExtensionName: extensionNameAzureTTS, Property: "azure_synthesis_voice_name"}, {ExtensionName: extensionNameElevenlabsTTS, Property: "voice_id"}, }, + "WorkerHttpServerPort": { + {ExtensionName: extensionNameHttpServer, Property: "listen_port"}, + }, } // Map the voice name to the voice type diff --git a/server/internal/http_client.go b/server/internal/http_client.go new file mode 100644 index 00000000..310e518d --- /dev/null +++ b/server/internal/http_client.go @@ -0,0 +1,15 @@ +package internal + +import ( + "crypto/tls" + "time" + + "github.com/go-resty/resty/v2" +) + +var ( + HttpClient = resty.New(). + SetRetryCount(0). + SetTimeout(5 * time.Second). + SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) +) diff --git a/server/internal/http_server.go b/server/internal/http_server.go index 13461156..7cbf158d 100644 --- a/server/internal/http_server.go +++ b/server/internal/http_server.go @@ -1,7 +1,7 @@ /** * * Agora Real Time Engagement - * Created by lixinhui in 2024. + * Created by Xinhui Li in 2024. * Copyright (c) 2024 Agora IO. All rights reserved. * */ @@ -10,8 +10,10 @@ package internal import ( "fmt" "log/slog" + "mime/multipart" "net/http" "os" + "path/filepath" "strings" "time" @@ -44,13 +46,14 @@ type PingReq struct { } type StartReq struct { - RequestId string `json:"request_id,omitempty"` - AgoraAsrLanguage string `json:"agora_asr_language,omitempty"` - ChannelName string `json:"channel_name,omitempty"` - GraphName string `json:"graph_name,omitempty"` - RemoteStreamId uint32 `json:"remote_stream_id,omitempty"` - Token string `json:"token,omitempty"` - VoiceType string `json:"voice_type,omitempty"` + RequestId string `json:"request_id,omitempty"` + AgoraAsrLanguage string `json:"agora_asr_language,omitempty"` + ChannelName string `json:"channel_name,omitempty"` + GraphName string `json:"graph_name,omitempty"` + RemoteStreamId uint32 `json:"remote_stream_id,omitempty"` + Token string `json:"token,omitempty"` + VoiceType string `json:"voice_type,omitempty"` + WorkerHttpServerPort int32 `json:"worker_http_server_port,omitempty"` } type StopReq struct { @@ -64,6 +67,13 @@ type GenerateTokenReq struct { Uid uint32 `json:"uid,omitempty"` } +type UploadReq struct { + RequestId string `form:"request_id,omitempty" json:"request_id,omitempty"` + ChannelName string `form:"channel_name,omitempty" json:"channel_name,omitempty"` + File *multipart.FileHeader `form:"file" binding:"required"` + Uid uint32 `form:"uid,omitempty" json:"uid,omitempty"` +} + func NewHttpServer(httpServerConfig *HttpServerConfig) *HttpServer { return &HttpServer{ config: httpServerConfig, @@ -136,6 +146,7 @@ func (s *HttpServer) handlerStart(c *gin.Context) { return } + req.WorkerHttpServerPort = getHttpServerPort() propertyJsonFile, logFile, err := s.processProperty(&req) if err != nil { slog.Error("handlerStart process property", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) @@ -144,6 +155,7 @@ func (s *HttpServer) handlerStart(c *gin.Context) { } worker := newWorker(req.ChannelName, logFile, propertyJsonFile) + worker.HttpServerPort = req.WorkerHttpServerPort worker.QuitTimeoutSeconds = s.config.WorkerQuitTimeoutSeconds if err := worker.start(&req); err != nil { slog.Error("handlerStart start worker failed", "err", err, "requestId", req.RequestId, logTag) @@ -223,6 +235,52 @@ func (s *HttpServer) handlerGenerateToken(c *gin.Context) { s.output(c, codeSuccess, map[string]any{"appId": s.config.AppId, "token": token, "channel_name": req.ChannelName, "uid": req.Uid}) } +func (s *HttpServer) handlerUpload(c *gin.Context) { + var req UploadReq + + if err := c.ShouldBind(&req); err != nil { + slog.Error("handlerUpload params invalid", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + s.output(c, codeErrParamsInvalid, http.StatusBadRequest) + return + } + + if !workers.Contains(req.ChannelName) { + slog.Error("handlerUpload channel not existed", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + s.output(c, codeErrChannelNotExisted, http.StatusBadRequest) + return + } + + slog.Info("handlerUpload start", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + + file := req.File + uploadFile := fmt.Sprintf("%s/file-%s-%d%s", s.config.LogPath, gmd5.MustEncryptString(req.ChannelName), time.Now().UnixNano(), filepath.Ext(file.Filename)) + if err := c.SaveUploadedFile(file, uploadFile); err != nil { + slog.Error("handlerUpload save file failed", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + s.output(c, codeErrSaveFileFailed, http.StatusBadRequest) + return + } + + // update worker + worker := workers.Get(req.ChannelName).(*Worker) + err := worker.update(&WorkerUpdateReq{ + RequestId: req.RequestId, + ChannelName: req.ChannelName, + Path: uploadFile, + Rte: &WorkerUpdateReqRte{ + Name: "file_download", + Type: "cmd", + }, + }) + if err != nil { + slog.Error("handlerUpload update worker failed", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + s.output(c, codeErrUpdateWorkerFailed, http.StatusBadRequest) + return + } + + slog.Info("handlerUpload end", "channelName", req.ChannelName, "uploadFile", uploadFile, "requestId", req.RequestId, logTag) + s.output(c, codeSuccess, map[string]any{"channel_name": req.ChannelName, "uid": req.Uid}) +} + func (s *HttpServer) output(c *gin.Context, code *Code, data any, httpStatus ...int) { if len(httpStatus) == 0 { httpStatus = append(httpStatus, http.StatusOK) @@ -291,6 +349,7 @@ func (s *HttpServer) Start() { r.POST("/start", s.handlerStart) r.POST("/stop", s.handlerStop) r.POST("/token/generate", s.handlerGenerateToken) + r.POST("/upload", s.handlerUpload) slog.Info("server start", "port", s.config.Port, logTag) diff --git a/server/internal/worker.go b/server/internal/worker.go index c250c296..c87e6981 100644 --- a/server/internal/worker.go +++ b/server/internal/worker.go @@ -3,18 +3,22 @@ package internal import ( "fmt" "log/slog" + "net/http" "os/exec" "strconv" "strings" + "sync/atomic" "syscall" "time" + "github.com/go-resty/resty/v2" "github.com/gogf/gf/container/gmap" "github.com/google/uuid" ) type Worker struct { ChannelName string + HttpServerPort int32 LogFile string PropertyJsonFile string Pid int @@ -23,13 +27,29 @@ type Worker struct { UpdateTs int64 } +type WorkerUpdateReq struct { + RequestId string `form:"request_id,omitempty" json:"request_id,omitempty"` + ChannelName string `form:"channel_name,omitempty" json:"channel_name,omitempty"` + Path string `form:"path,omitempty" json:"path,omitempty"` + Rte *WorkerUpdateReqRte `form:"rte,omitempty" json:"rte,omitempty"` +} + +type WorkerUpdateReqRte struct { + Name string `form:"name,omitempty" json:"name,omitempty"` + Type string `form:"type,omitempty" json:"type,omitempty"` +} + const ( workerCleanSleepSeconds = 5 workerExec = "/app/agents/bin/worker" + workerHttpServerUrl = "http://127.0.0.1" ) var ( - workers = gmap.New(true) + workers = gmap.New(true) + httpServerPort = httpServerPortMin + httpServerPortMin = int32(10000) + httpServerPortMax = int32(30000) ) func newWorker(channelName string, logFile string, propertyJsonFile string) *Worker { @@ -43,6 +63,15 @@ func newWorker(channelName string, logFile string, propertyJsonFile string) *Wor } } +func getHttpServerPort() int32 { + if atomic.LoadInt32(&httpServerPort) > httpServerPortMax { + atomic.StoreInt32(&httpServerPort, httpServerPortMin) + } + + atomic.AddInt32(&httpServerPort, 1) + return httpServerPort +} + func (w *Worker) start(req *StartReq) (err error) { shell := fmt.Sprintf("cd /app/agents && nohup %s --property %s > %s 2>&1 &", workerExec, w.PropertyJsonFile, w.LogFile) slog.Info("Worker start", "requestId", req.RequestId, "shell", shell, logTag) @@ -84,6 +113,34 @@ func (w *Worker) stop(requestId string, channelName string) (err error) { return } +func (w *Worker) update(req *WorkerUpdateReq) (err error) { + slog.Info("Worker update start", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + + var res *resty.Response + + defer func() { + if err != nil { + slog.Error("Worker update error", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + } + }() + + workerUpdateUrl := fmt.Sprintf("%s:%d/cmd", workerHttpServerUrl, w.HttpServerPort) + res, err = HttpClient.R(). + SetHeader("Content-Type", "application/json"). + SetBody(req). + Post(workerUpdateUrl) + if err != nil { + return + } + + if res.StatusCode() != http.StatusOK { + return fmt.Errorf("%s, status: %d", codeErrHttpStatusNotOk.msg, res.StatusCode()) + } + + slog.Info("Worker update end", "channelName", req.ChannelName, "worker", w, "requestId", req.RequestId, logTag) + return +} + func cleanWorker() { for { for _, channelName := range workers.Keys() { From f175b8cd002ceb8e5c2864f0494261b39341487a Mon Sep 17 00:00:00 2001 From: sunshinexcode <24xinhui@163.com> Date: Mon, 12 Aug 2024 18:33:59 +0800 Subject: [PATCH 14/34] chore(): add api for /vector/document (#193) * chore(): add api for /vector/document * fix(): fix mapping --- .env.example | 18 +++++- agents/property.json.example | 14 ++--- server/internal/code.go | 1 + server/internal/config.go | 62 +++++++++++++++----- server/internal/http_server.go | 100 ++++++++++++++++++++++++++++----- server/internal/worker.go | 2 + server/main.go | 11 +++- 7 files changed, 171 insertions(+), 37 deletions(-) diff --git a/.env.example b/.env.example index 17141464..cbd296eb 100644 --- a/.env.example +++ b/.env.example @@ -7,9 +7,12 @@ # ------------------------------ # Log path -LOG_PATH=/tmp +LOG_PATH=/tmp/astra # Graph designer server port GRAPH_DESIGNER_SERVER_PORT=49483 +# The corresponding graph name based on the language +GRAPH_NAME_ZH=va.openai.azure +GRAPH_NAME_EN=va.openai.azure # Server port SERVER_PORT=8080 # Maximum number of workers @@ -26,6 +29,19 @@ AGORA_APP_CERTIFICATE= # Worker Configuration # ------------------------------ +# Extension: aliyun_analyticdb_vector_storage +ALIBABA_CLOUD_ACCESS_KEY_ID= +ALIBABA_CLOUD_ACCESS_KEY_SECRET= +ALIYUN_ANALYTICDB_ACCOUNT= +ALIYUN_ANALYTICDB_ACCOUNT_PASSWORD= +ALIYUN_ANALYTICDB_INSTANCE_ID= +ALIYUN_ANALYTICDB_INSTANCE_REGION=cn-shanghai +ALIYUN_ANALYTICDB_NAMESPACE= +ALIYUN_ANALYTICDB_NAMESPACE_PASSWORD= + +# Extension: aliyun_text_embedding +ALIYUN_TEXT_EMBEDDING_API_KEY= + # Extension: bedrock_llm # Extension: polly_tts AWS_ACCESS_KEY_ID= diff --git a/agents/property.json.example b/agents/property.json.example index c63abb40..10907ba6 100644 --- a/agents/property.json.example +++ b/agents/property.json.example @@ -1465,7 +1465,7 @@ "type": "extension", "extension_group": "embedding", "addon": "aliyun_text_embedding", - "name": "embedding", + "name": "aliyun_text_embedding", "property": { "api_key": "", "model": "text-embedding-v3" @@ -1475,7 +1475,7 @@ "type": "extension", "extension_group": "vector_storage", "addon": "aliyun_analyticdb_vector_storage", - "name": "vector_storage", + "name": "aliyun_analyticdb_vector_storage", "property": { "alibaba_cloud_access_key_id": "", "alibaba_cloud_access_key_secret": "", @@ -1617,7 +1617,7 @@ "dest": [ { "extension_group": "embedding", - "extension": "embedding" + "extension": "aliyun_text_embedding" } ] }, @@ -1626,7 +1626,7 @@ "dest": [ { "extension_group": "vector_storage", - "extension": "vector_storage" + "extension": "aliyun_analyticdb_vector_storage" } ] } @@ -1701,7 +1701,7 @@ "dest": [ { "extension_group": "embedding", - "extension": "embedding" + "extension": "aliyun_text_embedding" } ] }, @@ -1710,7 +1710,7 @@ "dest": [ { "extension_group": "vector_storage", - "extension": "vector_storage" + "extension": "aliyun_analyticdb_vector_storage" } ] }, @@ -1719,7 +1719,7 @@ "dest": [ { "extension_group": "vector_storage", - "extension": "vector_storage" + "extension": "aliyun_analyticdb_vector_storage" } ] }, diff --git a/server/internal/code.go b/server/internal/code.go index 1a583ee0..8a4b61b2 100644 --- a/server/internal/code.go +++ b/server/internal/code.go @@ -16,6 +16,7 @@ var ( codeErrChannelEmpty = NewCode("10004", "channel empty") codeErrGenerateTokenFailed = NewCode("10005", "generate token failed") codeErrSaveFileFailed = NewCode("10006", "save file failed") + codeErrParseJsonFailed = NewCode("10007", "parse json failed") codeErrProcessPropertyFailed = NewCode("10100", "process property json failed") codeErrStartWorkerFailed = NewCode("10101", "start worker failed") diff --git a/server/internal/config.go b/server/internal/config.go index 9297e300..f1cf6dca 100644 --- a/server/internal/config.go +++ b/server/internal/config.go @@ -1,6 +1,9 @@ package internal -import "log/slog" +import ( + "log/slog" + "os" +) type Prop struct { ExtensionName string @@ -9,22 +12,22 @@ type Prop struct { const ( // Extension name - extensionNameAgoraRTC = "agora_rtc" - extensionNameAzureTTS = "azure_tts" - extensionNameBedrockLLM = "bedrock_llm" - extensionNameCosyTTS = "cosy_tts" - extensionNameElevenlabsTTS = "elevenlabs_tts" - extensionNameHttpServer = "http_server" - extensionNameLiteLLM = "litellm" - extensionNameOpenaiChatgpt = "openai_chatgpt" - extensionNamePollyTTS = "polly_tts" - extensionNameQwenLLM = "qwen_llm" + extensionNameAgoraRTC = "agora_rtc" + extensionNameAliyunAnalyticdbVectorStorage = "aliyun_analyticdb_vector_storage" + extensionNameAliyunTextEmbedding = "aliyun_text_embedding" + extensionNameAzureTTS = "azure_tts" + extensionNameBedrockLLM = "bedrock_llm" + extensionNameCosyTTS = "cosy_tts" + extensionNameElevenlabsTTS = "elevenlabs_tts" + extensionNameHttpServer = "http_server" + extensionNameLiteLLM = "litellm" + extensionNameOpenaiChatgpt = "openai_chatgpt" + extensionNamePollyTTS = "polly_tts" + extensionNameQwenLLM = "qwen_llm" // Language languageChinese = "zh-CN" languageEnglish = "en-US" - // Default graph name - graphNameDefault = "va.openai.azure" // Property json PropertyJsonFile = "./agents/property.json" // Token expire time @@ -42,6 +45,33 @@ var ( "AGORA_APP_ID": { {ExtensionName: extensionNameAgoraRTC, Property: "app_id"}, }, + "ALIBABA_CLOUD_ACCESS_KEY_ID": { + {ExtensionName: extensionNameAliyunAnalyticdbVectorStorage, Property: "alibaba_cloud_access_key_id"}, + }, + "ALIBABA_CLOUD_ACCESS_KEY_SECRET": { + {ExtensionName: extensionNameAliyunAnalyticdbVectorStorage, Property: "alibaba_cloud_access_key_secret"}, + }, + "ALIYUN_ANALYTICDB_ACCOUNT": { + {ExtensionName: extensionNameAliyunAnalyticdbVectorStorage, Property: "adbpg_account"}, + }, + "ALIYUN_ANALYTICDB_ACCOUNT_PASSWORD": { + {ExtensionName: extensionNameAliyunAnalyticdbVectorStorage, Property: "adbpg_account_password"}, + }, + "ALIYUN_ANALYTICDB_INSTANCE_ID": { + {ExtensionName: extensionNameAliyunAnalyticdbVectorStorage, Property: "adbpg_instance_id"}, + }, + "ALIYUN_ANALYTICDB_INSTANCE_REGION": { + {ExtensionName: extensionNameAliyunAnalyticdbVectorStorage, Property: "adbpg_instance_region"}, + }, + "ALIYUN_ANALYTICDB_NAMESPACE": { + {ExtensionName: extensionNameAliyunAnalyticdbVectorStorage, Property: "adbpg_namespace"}, + }, + "ALIYUN_ANALYTICDB_NAMESPACE_PASSWORD": { + {ExtensionName: extensionNameAliyunAnalyticdbVectorStorage, Property: "adbpg_namespace_password"}, + }, + "ALIYUN_TEXT_EMBEDDING_API_KEY": { + {ExtensionName: extensionNameAliyunTextEmbedding, Property: "api_key"}, + }, "AWS_ACCESS_KEY_ID": { {ExtensionName: extensionNameBedrockLLM, Property: "access_key"}, {ExtensionName: extensionNamePollyTTS, Property: "access_key"}, @@ -99,6 +129,12 @@ var ( }, } + // The corresponding graph name based on the language + graphNameMap = map[string]string{ + languageChinese: os.Getenv("GRAPH_NAME_ZH"), + languageEnglish: os.Getenv("GRAPH_NAME_EN"), + } + // Retrieve parameters from the request and map them to the property.json file startPropMap = map[string][]Prop{ "AgoraAsrLanguage": { diff --git a/server/internal/http_server.go b/server/internal/http_server.go index 7cbf158d..75d6fb8c 100644 --- a/server/internal/http_server.go +++ b/server/internal/http_server.go @@ -1,13 +1,14 @@ /** * * Agora Real Time Engagement - * Created by Xinhui Li in 2024. + * Created by XinHui Li in 2024. * Copyright (c) 2024 Agora IO. All rights reserved. * */ package internal import ( + "encoding/json" "fmt" "log/slog" "mime/multipart" @@ -67,11 +68,17 @@ type GenerateTokenReq struct { Uid uint32 `json:"uid,omitempty"` } -type UploadReq struct { +type VectorDocumentUpdate struct { + RequestId string `json:"request_id,omitempty"` + ChannelName string `json:"channel_name,omitempty"` + Collection string `json:"collection,omitempty"` + FileName string `json:"file_name,omitempty"` +} + +type VectorDocumentUpload struct { RequestId string `form:"request_id,omitempty" json:"request_id,omitempty"` ChannelName string `form:"channel_name,omitempty" json:"channel_name,omitempty"` File *multipart.FileHeader `form:"file" binding:"required"` - Uid uint32 `form:"uid,omitempty" json:"uid,omitempty"` } func NewHttpServer(httpServerConfig *HttpServerConfig) *HttpServer { @@ -235,50 +242,111 @@ func (s *HttpServer) handlerGenerateToken(c *gin.Context) { s.output(c, codeSuccess, map[string]any{"appId": s.config.AppId, "token": token, "channel_name": req.ChannelName, "uid": req.Uid}) } -func (s *HttpServer) handlerUpload(c *gin.Context) { - var req UploadReq +func (s *HttpServer) handlerVectorDocumentPresetList(c *gin.Context) { + presetList := []map[string]any{} + vectorDocumentPresetList := os.Getenv("VECTOR_DOCUMENT_PRESET_LIST") + + if vectorDocumentPresetList != "" { + err := json.Unmarshal([]byte(vectorDocumentPresetList), &presetList) + if err != nil { + slog.Error("handlerVectorDocumentPresetList parse json failed", "err", err, logTag) + s.output(c, codeErrParseJsonFailed, http.StatusBadRequest) + return + } + } + + s.output(c, codeSuccess, presetList) +} + +func (s *HttpServer) handlerVectorDocumentUpdate(c *gin.Context) { + var req VectorDocumentUpdate if err := c.ShouldBind(&req); err != nil { - slog.Error("handlerUpload params invalid", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + slog.Error("handlerVectorDocumentUpdate params invalid", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) s.output(c, codeErrParamsInvalid, http.StatusBadRequest) return } if !workers.Contains(req.ChannelName) { - slog.Error("handlerUpload channel not existed", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + slog.Error("handlerVectorDocumentUpdate channel not existed", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) s.output(c, codeErrChannelNotExisted, http.StatusBadRequest) return } - slog.Info("handlerUpload start", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + slog.Info("handlerVectorDocumentUpdate start", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + + // update worker + worker := workers.Get(req.ChannelName).(*Worker) + err := worker.update(&WorkerUpdateReq{ + RequestId: req.RequestId, + ChannelName: req.ChannelName, + Collection: req.Collection, + FileName: req.FileName, + Rte: &WorkerUpdateReqRte{ + Name: "update_querying_collection", + Type: "cmd", + }, + }) + if err != nil { + slog.Error("handlerVectorDocumentUpdate update worker failed", "err", err, "channelName", req.ChannelName, "Collection", req.Collection, "FileName", req.FileName, "requestId", req.RequestId, logTag) + s.output(c, codeErrUpdateWorkerFailed, http.StatusBadRequest) + return + } + + slog.Info("handlerVectorDocumentUpdate end", "channelName", req.ChannelName, "Collection", req.Collection, "FileName", req.FileName, "requestId", req.RequestId, logTag) + s.output(c, codeSuccess, map[string]any{"channel_name": req.ChannelName}) +} + +func (s *HttpServer) handlerVectorDocumentUpload(c *gin.Context) { + var req VectorDocumentUpload + + if err := c.ShouldBind(&req); err != nil { + slog.Error("handlerVectorDocumentUpload params invalid", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + s.output(c, codeErrParamsInvalid, http.StatusBadRequest) + return + } + + if !workers.Contains(req.ChannelName) { + slog.Error("handlerVectorDocumentUpload channel not existed", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + s.output(c, codeErrChannelNotExisted, http.StatusBadRequest) + return + } + + slog.Info("handlerVectorDocumentUpload start", "channelName", req.ChannelName, "requestId", req.RequestId, logTag) file := req.File uploadFile := fmt.Sprintf("%s/file-%s-%d%s", s.config.LogPath, gmd5.MustEncryptString(req.ChannelName), time.Now().UnixNano(), filepath.Ext(file.Filename)) if err := c.SaveUploadedFile(file, uploadFile); err != nil { - slog.Error("handlerUpload save file failed", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + slog.Error("handlerVectorDocumentUpload save file failed", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) s.output(c, codeErrSaveFileFailed, http.StatusBadRequest) return } + // Generate collection + collection := fmt.Sprintf("%s_%d", gmd5.MustEncryptString(req.ChannelName), time.Now().UnixNano()) + fileName := filepath.Base(file.Filename) + // update worker worker := workers.Get(req.ChannelName).(*Worker) err := worker.update(&WorkerUpdateReq{ RequestId: req.RequestId, ChannelName: req.ChannelName, + Collection: collection, + FileName: fileName, Path: uploadFile, Rte: &WorkerUpdateReqRte{ - Name: "file_download", + Name: "file_chunk", Type: "cmd", }, }) if err != nil { - slog.Error("handlerUpload update worker failed", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) + slog.Error("handlerVectorDocumentUpload update worker failed", "err", err, "channelName", req.ChannelName, "requestId", req.RequestId, logTag) s.output(c, codeErrUpdateWorkerFailed, http.StatusBadRequest) return } - slog.Info("handlerUpload end", "channelName", req.ChannelName, "uploadFile", uploadFile, "requestId", req.RequestId, logTag) - s.output(c, codeSuccess, map[string]any{"channel_name": req.ChannelName, "uid": req.Uid}) + slog.Info("handlerVectorDocumentUpload end", "channelName", req.ChannelName, "collection", collection, "uploadFile", uploadFile, "requestId", req.RequestId, logTag) + s.output(c, codeSuccess, map[string]any{"channel_name": req.ChannelName, "collection": collection, "file_name": fileName}) } func (s *HttpServer) output(c *gin.Context, code *Code, data any, httpStatus ...int) { @@ -301,7 +369,7 @@ func (s *HttpServer) processProperty(req *StartReq) (propertyJsonFile string, lo // Get graph name graphName := req.GraphName if graphName == "" { - graphName = graphNameDefault + graphName = graphNameMap[req.AgoraAsrLanguage] } // Generate token @@ -349,7 +417,9 @@ func (s *HttpServer) Start() { r.POST("/start", s.handlerStart) r.POST("/stop", s.handlerStop) r.POST("/token/generate", s.handlerGenerateToken) - r.POST("/upload", s.handlerUpload) + r.GET("/vector/document/preset/list", s.handlerVectorDocumentPresetList) + r.POST("/vector/document/update", s.handlerVectorDocumentUpdate) + r.POST("/vector/document/upload", s.handlerVectorDocumentUpload) slog.Info("server start", "port", s.config.Port, logTag) diff --git a/server/internal/worker.go b/server/internal/worker.go index c87e6981..583e5aa7 100644 --- a/server/internal/worker.go +++ b/server/internal/worker.go @@ -30,6 +30,8 @@ type Worker struct { type WorkerUpdateReq struct { RequestId string `form:"request_id,omitempty" json:"request_id,omitempty"` ChannelName string `form:"channel_name,omitempty" json:"channel_name,omitempty"` + Collection string `form:"collection,omitempty" json:"collection,omitempty"` + FileName string `form:"filename,omitempty" json:"filename,omitempty"` Path string `form:"path,omitempty" json:"path,omitempty"` Rte *WorkerUpdateReqRte `form:"rte,omitempty" json:"rte,omitempty"` } diff --git a/server/main.go b/server/main.go index 442e6506..d590b89d 100644 --- a/server/main.go +++ b/server/main.go @@ -20,6 +20,15 @@ func main() { slog.Warn("load .env file failed", "err", err) } + // Check if the directory exists + logPath := os.Getenv("LOG_PATH") + if _, err := os.Stat(logPath); os.IsNotExist(err) { + if err := os.MkdirAll(logPath, os.ModePerm); err != nil { + slog.Error("create log directory failed", "err", err) + os.Exit(1) + } + } + // Check environment agoraAppId := os.Getenv("AGORA_APP_ID") if len(agoraAppId) != 32 { @@ -49,7 +58,7 @@ func main() { httpServerConfig := &internal.HttpServerConfig{ AppId: agoraAppId, AppCertificate: os.Getenv("AGORA_APP_CERTIFICATE"), - LogPath: os.Getenv("LOG_PATH"), + LogPath: logPath, Port: os.Getenv("SERVER_PORT"), WorkersMax: workersMax, WorkerQuitTimeoutSeconds: workerQuitTimeoutSeconds, From 18d967c6bbef9570c387a6bccd6e9b80dc87e970 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Mon, 12 Aug 2024 04:30:44 -0700 Subject: [PATCH 15/34] feat: update querying collection --- .../file_chunker/file_chunker_extension.py | 38 +++++++++------ .../extension/file_chunker/manifest.json | 12 +++-- .../llama_index_chat_engine/extension.py | 47 ++++++++++++++++--- .../llama_index_chat_engine/manifest.json | 21 +++++++-- agents/property.json.example | 11 ++++- 5 files changed, 101 insertions(+), 28 deletions(-) diff --git a/agents/addon/extension/file_chunker/file_chunker_extension.py b/agents/addon/extension/file_chunker/file_chunker_extension.py index d1fb4e02..b9048988 100644 --- a/agents/addon/extension/file_chunker/file_chunker_extension.py +++ b/agents/addon/extension/file_chunker/file_chunker_extension.py @@ -21,7 +21,7 @@ import uuid, math import queue, threading -CMD_FILE_DOWNLOADED = "file_downloaded" +CMD_FILE_CHUNK = "file_chunk" UPSERT_VECTOR_CMD = "upsert_vector" FILE_CHUNKED_CMD = "file_chunked" @@ -147,7 +147,7 @@ def file_chunked(self, rte: RteEnv, path: str): ) cmd_out = Cmd.create(FILE_CHUNKED_CMD) cmd_out.set_property_string("path", path) - cmd_out.set_property_string("collection_name", self.new_collection_name) + cmd_out.set_property_string("collection", self.new_collection_name) rte.send_cmd( cmd_out, lambda rte, result: logger.info("send_cmd done"), @@ -158,9 +158,16 @@ def file_chunked(self, rte: RteEnv, path: str): def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: cmd_name = cmd.get_name() - if cmd_name == CMD_FILE_DOWNLOADED: + if cmd_name == CMD_FILE_CHUNK: path = cmd.get_property_string("path") - self.queue.put(path) # make sure files are processed in order + + collection = None + try: + collection = cmd.get_property_string("collection") + except Exception as e: + logger.warning("missing collection property in cmd {}".format(cmd_name)) + + self.queue.put((path, collection)) # make sure files are processed in order else: logger.info("unknown cmd {}".format(cmd_name)) @@ -170,26 +177,27 @@ def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: def async_handler(self, rte: RteEnv) -> None: while not self.stop: - path = self.queue.get() - if path is None: + value = self.queue.get() + if value is None: break + path, collection = value # start processing the file start_time = datetime.now() - collection_name = self.generate_collection_name() - logger.info( - "start processing {}, collection_name {}".format(path, collection_name) - ) + if collection is None: + collection = self.generate_collection_name() + logger.info("collection {} generated".format(collection)) + logger.info("start processing {}, collection {}".format(path, collection)) # create collection - self.create_collection(rte, collection_name, True) - logger.info("collection_name {} created".format(collection_name)) + self.create_collection(rte, collection, True) + logger.info("collection {} created".format(collection)) # split nodes = self.split(path) # reset counters and events - self.new_collection_name = collection_name + self.new_collection_name = collection self.expected[path] = math.ceil(len(nodes) / BATCH_SIZE) self.counters[path] = 0 self.file_chunked_event.clear() @@ -202,9 +210,9 @@ def async_handler(self, rte: RteEnv) -> None: self.file_chunked_event.wait() logger.info( - "finished processing {}, collection_name {}, cost {}ms".format( + "finished processing {}, collection {}, cost {}ms".format( path, - collection_name, + collection, int((datetime.now() - start_time).total_seconds() * 1000), ) ) diff --git a/agents/addon/extension/file_chunker/manifest.json b/agents/addon/extension/file_chunker/manifest.json index 248114db..b491e55a 100644 --- a/agents/addon/extension/file_chunker/manifest.json +++ b/agents/addon/extension/file_chunker/manifest.json @@ -14,10 +14,16 @@ "property": {}, "cmd_in": [ { - "name": "file_downloaded", + "name": "file_chunk", "property": { + "filename": { + "type": "string" + }, "path": { "type": "string" + }, + "collection": { + "type": "string" } }, "required": [ @@ -86,13 +92,13 @@ "path": { "type": "string" }, - "collection_name": { + "collection": { "type": "string" } }, "required": [ "path", - "collection_name" + "collection" ] } ] diff --git a/agents/addon/extension/llama_index_chat_engine/extension.py b/agents/addon/extension/llama_index_chat_engine/extension.py index 2c189dc0..718d6bb9 100644 --- a/agents/addon/extension/llama_index_chat_engine/extension.py +++ b/agents/addon/extension/llama_index_chat_engine/extension.py @@ -103,14 +103,49 @@ def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: cmd_name = cmd.get_name() logger.info("on_cmd {}".format(cmd_name)) if cmd_name == "file_chunked": - coll = cmd.get_property_string("collection_name") - self.collection_name = coll + coll = cmd.get_property_string("collection") - file_chunked_text = "Your document has been processed. Please wait a moment while we process your document for you. " + # only update selected collection if empty + if len(self.collection_name) == 0: + logger.info( + "collection for querying has been updated from {} to {}".format( + self.collection_name, coll + ) + ) + self.collection_name = coll + else: + logger.info( + "new collection {} incoming but won't change current collection_name {}".format( + coll, self.collection_name + ) + ) + + # notify user + file_chunked_text = "Your document has been processed. You can now start asking questions about your document. " self._send_text_data(rte, file_chunked_text, True) - elif cmd_name == "file_downloaded": - file_downloaded_text = "Your document has been received. Please wait a moment while we process it for you. " - self._send_text_data(rte, file_downloaded_text, True) + elif cmd_name == "file_chunk": + self.collection_name = "" # clear current collection + + # notify user + file_chunk_text = "Your document has been received. Please wait a moment while we process it for you. " + self._send_text_data(rte, file_chunk_text, True) + elif cmd_name == "update_querying_collection": + coll = cmd.get_property_string("collection") + logger.info( + "collection for querying has been updated from {} to {}".format( + self.collection_name, coll + ) + ) + self.collection_name = coll + + # notify user + update_querying_collection_text = "Your document has been updated. " + if len(self.collection_name) > 0: + update_querying_collection_text += ( + "You can now start asking questions about your document. " + ) + self._send_text_data(rte, update_querying_collection_text, True) + elif cmd_name == "flush": self.flush() rte.send_cmd(Cmd.create("flush"), None) diff --git a/agents/addon/extension/llama_index_chat_engine/manifest.json b/agents/addon/extension/llama_index_chat_engine/manifest.json index 11ca8973..a7c9d62f 100644 --- a/agents/addon/extension/llama_index_chat_engine/manifest.json +++ b/agents/addon/extension/llama_index_chat_engine/manifest.json @@ -50,17 +50,32 @@ "name": "flush" }, { - "name": "file_downloaded" + "name": "file_chunk" }, { "name": "file_chunked", "property": { - "collection_name": { + "collection": { + "type": "string" + } + }, + "required": [ + "collection" + ] + }, + { + "name": "update_querying_collection", + "property": { + "filename": { + "type": "string" + }, + "collection": { "type": "string" } }, "required": [ - "collection_name" + "filename", + "collection" ] } ], diff --git a/agents/property.json.example b/agents/property.json.example index 10907ba6..209d6a00 100644 --- a/agents/property.json.example +++ b/agents/property.json.example @@ -1678,7 +1678,7 @@ "extension": "http_server", "cmd": [ { - "name": "file_downloaded", + "name": "file_chunk", "dest": [ { "extension_group": "file_chunker", @@ -1689,6 +1689,15 @@ "extension": "llama_index" } ] + }, + { + "name": "update_querying_collection", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] } ] }, From da165eec6d336fa888e674af8928b39e2b1ddefd Mon Sep 17 00:00:00 2001 From: sunshinexcode <24xinhui@163.com> Date: Mon, 12 Aug 2024 21:18:49 +0800 Subject: [PATCH 16/34] fix(): fix graphNameMap not being able to retrieve environment variable values (#195) --- docker-compose.yml | 21 ++------------------- server/internal/config.go | 14 ++++++++++++-- server/main.go | 3 +++ 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 86b95e91..2614e35f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,25 +14,8 @@ services: - ./:/app - ${LOG_PATH}:${LOG_PATH} working_dir: /app - environment: - AGORA_APP_ID: ${AGORA_APP_ID} - AGORA_APP_CERTIFICATE: ${AGORA_APP_CERTIFICATE} - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} - AWS_BEDROCK_MODEL: ${AWS_BEDROCK_MODEL} - AWS_REGION: ${AWS_REGION} - AZURE_STT_KEY: ${AZURE_STT_KEY} - AZURE_STT_REGION: ${AZURE_STT_REGION} - AZURE_TTS_KEY: ${AZURE_TTS_KEY} - AZURE_TTS_REGION: ${AZURE_TTS_REGION} - COSY_TTS_KEY: ${COSY_TTS_KEY} - ELEVENLABS_TTS_KEY: ${ELEVENLABS_TTS_KEY} - LITELLM_MODEL: ${LITELLM_MODEL} - OPENAI_API_KEY: ${OPENAI_API_KEY} - OPENAI_BASE_URL: ${OPENAI_BASE_URL} - OPENAI_MODEL: ${OPENAI_MODEL} - OPENAI_PROXY_URL: ${OPENAI_PROXY_URL} - QWEN_API_KEY: ${QWEN_API_KEY} + env_file: + - .env astra_playground: image: ghcr.io/rte-design/astra_playground:v0.2.0-15-g38e1738 container_name: astra_playground diff --git a/server/internal/config.go b/server/internal/config.go index f1cf6dca..ffbce550 100644 --- a/server/internal/config.go +++ b/server/internal/config.go @@ -131,8 +131,8 @@ var ( // The corresponding graph name based on the language graphNameMap = map[string]string{ - languageChinese: os.Getenv("GRAPH_NAME_ZH"), - languageEnglish: os.Getenv("GRAPH_NAME_EN"), + languageChinese: "va.openai.azure", + languageEnglish: "va.openai.azure", } // Retrieve parameters from the request and map them to the property.json file @@ -190,3 +190,13 @@ var ( }, } ) + +func SetGraphNameMap() { + if graphNameZH := os.Getenv("GRAPH_NAME_ZH"); graphNameZH != "" { + graphNameMap[languageChinese] = graphNameZH + } + + if graphNameEN := os.Getenv("GRAPH_NAME_EN"); graphNameEN != "" { + graphNameMap[languageEnglish] = graphNameEN + } +} diff --git a/server/main.go b/server/main.go index d590b89d..2bc4bd9c 100644 --- a/server/main.go +++ b/server/main.go @@ -48,6 +48,9 @@ func main() { os.Exit(1) } + // Set graph name map + internal.SetGraphNameMap() + // Process property.json if err = processProperty(internal.PropertyJsonFile); err != nil { slog.Error("process property.json failed", "err", err) From b1b77e046a89f7db4bccf102bff633df7398f320 Mon Sep 17 00:00:00 2001 From: sunshinexcode <24xinhui@163.com> Date: Tue, 13 Aug 2024 10:54:11 +0800 Subject: [PATCH 17/34] upgrade image version (#197) * fix(): fix graphNameMap not being able to retrieve environment variable values * chore(): upgrade image version --- .devcontainer/devcontainer.json | 2 +- Dockerfile | 2 +- docker-compose.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2cece602..6aec5d28 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ // README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile { "name": "astra", - "image": "ghcr.io/rte-design/astra_agents_build:0.3.5", + "image": "ghcr.io/rte-design/astra_agents_build:0.3.6", "customizations": { "vscode": { "extensions": [ diff --git a/Dockerfile b/Dockerfile index a383ca16..dfac37fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/rte-design/astra_agents_build:0.3.5 AS builder +FROM ghcr.io/rte-design/astra_agents_build:0.3.6 AS builder ARG SESSION_CONTROL_CONF=session_control.conf diff --git a/docker-compose.yml b/docker-compose.yml index 2614e35f..179790ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: astra_agents_dev: - image: ghcr.io/rte-design/astra_agents_build:0.3.5 + image: ghcr.io/rte-design/astra_agents_build:0.3.6 container_name: astra_agents_dev platform: linux/amd64 tty: true @@ -23,7 +23,7 @@ services: ports: - "3000:3000" astra_graph_designer: - image: agoraio/astra_graph_designer:0.1.0 + image: agoraio/astra_graph_designer:0.3.0 container_name: astra_graph_designer restart: always ports: From 51b73ed8d9c8b6f55126375ffa9ad766856ac0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A3=E8=87=BB?= Date: Tue, 13 Aug 2024 20:24:37 +0800 Subject: [PATCH 18/34] feat: web ui update (#198) --- playground/package.json | 7 +- playground/postcss.config.js | 5 + .../src/{css/globals.css => app/global.css} | 18 +- playground/src/app/home/index.module.scss | 70 - playground/src/app/home/page.tsx | 79 +- playground/src/app/index.module.scss | 4 +- playground/src/app/layout.tsx | 3 +- playground/src/assets/color_picker.svg | 17 + playground/src/assets/info.svg | 3 + playground/src/assets/logo.svg | 68 +- playground/src/assets/logo_small.svg | 33 + playground/src/assets/pdf.svg | 3 + playground/src/assets/voice.svg | 3 + playground/src/common/constant.ts | 18 +- playground/src/common/hooks.ts | 7 +- playground/src/common/request.ts | 36 + playground/src/common/utils.ts | 5 + .../chat/chatItem/index.module.scss | 62 - .../src/components/chat/chatItem/index.tsx | 41 - playground/src/components/chat/index.tsx | 32 - .../components/customSelect/index.module.scss | 22 + .../src/components/customSelect/index.tsx | 19 + playground/src/components/header/index.tsx | 38 - .../components/icons/colorPicker/index.tsx | 6 + playground/src/components/icons/index.tsx | 4 + .../src/components/icons/info/index.tsx | 6 + .../src/components/icons/logo/index.tsx | 4 +- playground/src/components/icons/pdf/index.tsx | 6 + playground/src/components/icons/types.ts | 2 + .../src/components/icons/voice/index.tsx | 6 + .../components/loginCard/index.module.scss | 2 +- playground/src/components/loginCard/index.tsx | 16 +- .../src/components/menu/index.module.scss | 24 - playground/src/components/menu/index.tsx | 53 - .../components/pdfSelect/index.module.scss | 8 + playground/src/components/pdfSelect/index.tsx | 72 + .../pdfSelect/upload/index.module.scss | 7 + .../src/components/pdfSelect/upload/index.tsx | 74 ++ .../src/components/setting/Info/index.tsx | 22 - .../src/components/setting/index.module.scss | 100 -- .../mobile/chat/chatItem/index.module.scss | 86 ++ .../platform/mobile/chat/chatItem/index.tsx | 50 + .../mobile}/chat/index.module.scss | 36 +- playground/src/platform/mobile/chat/index.tsx | 64 + .../mobile/description/index.module.scss | 71 + .../mobile/description}/index.tsx | 69 +- .../platform/mobile/entry/index.module.scss | 18 + .../src/platform/mobile/entry/index.tsx | 30 + .../platform/mobile/header/index.module.scss | 51 + .../src/platform/mobile/header/index.tsx | 46 + .../header/infoPopover}/index.module.scss | 11 +- .../mobile/header/infoPopover}/index.tsx | 36 +- .../mobile}/header/network/index.module.scss | 0 .../mobile}/header/network/index.tsx | 11 +- .../colorPicker/index.module.scss | 24 + .../header/stylePopover/colorPicker/index.tsx | 22 + .../header/stylePopover}/index.module.scss | 10 +- .../mobile/header/stylePopover}/index.tsx | 30 +- .../src/platform/mobile/menu/context.ts | 9 + .../platform/mobile/menu/index.module.scss | 69 + playground/src/platform/mobile/menu/index.tsx | 76 ++ .../mobile}/rtc/agent/index.module.scss | 0 .../mobile}/rtc/agent/index.tsx | 0 .../rtc/audioVisualizer/index.module.scss | 0 .../mobile}/rtc/audioVisualizer/index.tsx | 0 .../camSection/camSelect/index.module.scss | 0 .../rtc/camSection/camSelect/index.tsx | 0 .../mobile/rtc/camSection/index.module.scss | 54 + .../platform/mobile/rtc/camSection/index.tsx | 42 + .../mobile}/rtc/index.module.scss | 20 +- .../mobile}/rtc/index.tsx | 24 +- .../mobile/rtc/micSection/index.module.scss | 58 + .../platform/mobile/rtc/micSection/index.tsx | 70 + .../micSection/micSelect/index.module.scss | 0 .../rtc/micSection/micSelect/index.tsx | 0 .../rtc/streamPlayer/index.module.scss | 0 .../mobile}/rtc/streamPlayer/index.tsx | 0 .../rtc/streamPlayer/localStreamPlayer.tsx | 0 .../pc/chat/chatItem/index.module.scss | 90 ++ .../src/platform/pc/chat/chatItem/index.tsx | 51 + .../src/platform/pc/chat/index.module.scss | 73 ++ playground/src/platform/pc/chat/index.tsx | 59 + .../platform/pc/description/index.module.scss | 72 + .../src/platform/pc/description/index.tsx | 99 ++ .../src/platform/pc/entry/index.module.scss | 17 + playground/src/platform/pc/entry/index.tsx | 22 + .../pc}/header/index.module.scss | 12 +- playground/src/platform/pc/header/index.tsx | 46 + .../pc/header/infoPopover}/index.module.scss | 13 +- .../platform/pc/header/infoPopover/index.tsx | 57 + .../pc/header/network/index.module.scss | 0 .../src/platform/pc/header/network/index.tsx | 37 + .../colorPicker/index.module.scss | 24 + .../header/stylePopover/colorPicker/index.tsx | 22 + .../pc/header/stylePopover/index.module.scss | 51 + .../platform/pc/header/stylePopover/index.tsx | 54 + .../platform/pc/rtc/agent/index.module.scss | 31 + .../src/platform/pc/rtc/agent/index.tsx | 34 + .../pc/rtc/audioVisualizer/index.module.scss | 17 + .../platform/pc/rtc/audioVisualizer/index.tsx | 48 + .../camSection/camSelect/index.module.scss | 4 + .../pc/rtc/camSection/camSelect/index.tsx | 57 + .../pc}/rtc/camSection/index.module.scss | 0 .../pc}/rtc/camSection/index.tsx | 6 +- .../src/platform/pc/rtc/index.module.scss | 55 + playground/src/platform/pc/rtc/index.tsx | 128 ++ .../pc}/rtc/micSection/index.module.scss | 0 .../pc}/rtc/micSection/index.tsx | 2 +- .../micSection/micSelect/index.module.scss | 4 + .../pc/rtc/micSection/micSelect/index.tsx | 58 + .../pc/rtc/streamPlayer/index.module.scss | 6 + .../platform/pc/rtc/streamPlayer/index.tsx | 1 + .../pc/rtc/streamPlayer/localStreamPlayer.tsx | 46 + playground/src/protobuf/SttMessage001.js | 143 -- playground/src/protobuf/SttMessage002.js | 130 -- playground/src/protobuf/SttMessage003.js | 1161 ----------------- playground/src/store/reducers/global.ts | 18 +- playground/src/types/index.ts | 27 + 118 files changed, 2756 insertions(+), 2111 deletions(-) rename playground/src/{css/globals.css => app/global.css} (74%) delete mode 100644 playground/src/app/home/index.module.scss create mode 100644 playground/src/assets/color_picker.svg create mode 100644 playground/src/assets/info.svg create mode 100644 playground/src/assets/logo_small.svg create mode 100644 playground/src/assets/pdf.svg create mode 100644 playground/src/assets/voice.svg delete mode 100644 playground/src/components/chat/chatItem/index.module.scss delete mode 100644 playground/src/components/chat/chatItem/index.tsx delete mode 100644 playground/src/components/chat/index.tsx create mode 100644 playground/src/components/customSelect/index.module.scss create mode 100644 playground/src/components/customSelect/index.tsx delete mode 100644 playground/src/components/header/index.tsx create mode 100644 playground/src/components/icons/colorPicker/index.tsx create mode 100644 playground/src/components/icons/info/index.tsx create mode 100644 playground/src/components/icons/pdf/index.tsx create mode 100644 playground/src/components/icons/voice/index.tsx delete mode 100644 playground/src/components/menu/index.module.scss delete mode 100644 playground/src/components/menu/index.tsx create mode 100644 playground/src/components/pdfSelect/index.module.scss create mode 100644 playground/src/components/pdfSelect/index.tsx create mode 100644 playground/src/components/pdfSelect/upload/index.module.scss create mode 100644 playground/src/components/pdfSelect/upload/index.tsx delete mode 100644 playground/src/components/setting/Info/index.tsx delete mode 100644 playground/src/components/setting/index.module.scss create mode 100644 playground/src/platform/mobile/chat/chatItem/index.module.scss create mode 100644 playground/src/platform/mobile/chat/chatItem/index.tsx rename playground/src/{components => platform/mobile}/chat/index.module.scss (76%) create mode 100644 playground/src/platform/mobile/chat/index.tsx create mode 100644 playground/src/platform/mobile/description/index.module.scss rename playground/src/{components/setting => platform/mobile/description}/index.tsx (50%) create mode 100644 playground/src/platform/mobile/entry/index.module.scss create mode 100644 playground/src/platform/mobile/entry/index.tsx create mode 100644 playground/src/platform/mobile/header/index.module.scss create mode 100644 playground/src/platform/mobile/header/index.tsx rename playground/src/{components/setting/Info => platform/mobile/header/infoPopover}/index.module.scss (84%) rename playground/src/{components/setting/status => platform/mobile/header/infoPopover}/index.tsx (51%) rename playground/src/{components => platform/mobile}/header/network/index.module.scss (100%) rename playground/src/{components => platform/mobile}/header/network/index.tsx (78%) create mode 100644 playground/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss create mode 100644 playground/src/platform/mobile/header/stylePopover/colorPicker/index.tsx rename playground/src/{components/setting/themeSelect => platform/mobile/header/stylePopover}/index.module.scss (92%) rename playground/src/{components/setting/themeSelect => platform/mobile/header/stylePopover}/index.tsx (61%) create mode 100644 playground/src/platform/mobile/menu/context.ts create mode 100644 playground/src/platform/mobile/menu/index.module.scss create mode 100644 playground/src/platform/mobile/menu/index.tsx rename playground/src/{components => platform/mobile}/rtc/agent/index.module.scss (100%) rename playground/src/{components => platform/mobile}/rtc/agent/index.tsx (100%) rename playground/src/{components => platform/mobile}/rtc/audioVisualizer/index.module.scss (100%) rename playground/src/{components => platform/mobile}/rtc/audioVisualizer/index.tsx (100%) rename playground/src/{components => platform/mobile}/rtc/camSection/camSelect/index.module.scss (100%) rename playground/src/{components => platform/mobile}/rtc/camSection/camSelect/index.tsx (100%) create mode 100644 playground/src/platform/mobile/rtc/camSection/index.module.scss create mode 100644 playground/src/platform/mobile/rtc/camSection/index.tsx rename playground/src/{components => platform/mobile}/rtc/index.module.scss (79%) rename playground/src/{components => platform/mobile}/rtc/index.tsx (76%) create mode 100644 playground/src/platform/mobile/rtc/micSection/index.module.scss create mode 100644 playground/src/platform/mobile/rtc/micSection/index.tsx rename playground/src/{components => platform/mobile}/rtc/micSection/micSelect/index.module.scss (100%) rename playground/src/{components => platform/mobile}/rtc/micSection/micSelect/index.tsx (100%) rename playground/src/{components => platform/mobile}/rtc/streamPlayer/index.module.scss (100%) rename playground/src/{components => platform/mobile}/rtc/streamPlayer/index.tsx (100%) rename playground/src/{components => platform/mobile}/rtc/streamPlayer/localStreamPlayer.tsx (100%) create mode 100644 playground/src/platform/pc/chat/chatItem/index.module.scss create mode 100644 playground/src/platform/pc/chat/chatItem/index.tsx create mode 100644 playground/src/platform/pc/chat/index.module.scss create mode 100644 playground/src/platform/pc/chat/index.tsx create mode 100644 playground/src/platform/pc/description/index.module.scss create mode 100644 playground/src/platform/pc/description/index.tsx create mode 100644 playground/src/platform/pc/entry/index.module.scss create mode 100644 playground/src/platform/pc/entry/index.tsx rename playground/src/{components => platform/pc}/header/index.module.scss (87%) create mode 100644 playground/src/platform/pc/header/index.tsx rename playground/src/{components/setting/status => platform/pc/header/infoPopover}/index.module.scss (83%) create mode 100644 playground/src/platform/pc/header/infoPopover/index.tsx create mode 100644 playground/src/platform/pc/header/network/index.module.scss create mode 100644 playground/src/platform/pc/header/network/index.tsx create mode 100644 playground/src/platform/pc/header/stylePopover/colorPicker/index.module.scss create mode 100644 playground/src/platform/pc/header/stylePopover/colorPicker/index.tsx create mode 100644 playground/src/platform/pc/header/stylePopover/index.module.scss create mode 100644 playground/src/platform/pc/header/stylePopover/index.tsx create mode 100644 playground/src/platform/pc/rtc/agent/index.module.scss create mode 100644 playground/src/platform/pc/rtc/agent/index.tsx create mode 100644 playground/src/platform/pc/rtc/audioVisualizer/index.module.scss create mode 100644 playground/src/platform/pc/rtc/audioVisualizer/index.tsx create mode 100644 playground/src/platform/pc/rtc/camSection/camSelect/index.module.scss create mode 100644 playground/src/platform/pc/rtc/camSection/camSelect/index.tsx rename playground/src/{components => platform/pc}/rtc/camSection/index.module.scss (100%) rename playground/src/{components => platform/pc}/rtc/camSection/index.tsx (94%) create mode 100644 playground/src/platform/pc/rtc/index.module.scss create mode 100644 playground/src/platform/pc/rtc/index.tsx rename playground/src/{components => platform/pc}/rtc/micSection/index.module.scss (100%) rename playground/src/{components => platform/pc}/rtc/micSection/index.tsx (98%) create mode 100644 playground/src/platform/pc/rtc/micSection/micSelect/index.module.scss create mode 100644 playground/src/platform/pc/rtc/micSection/micSelect/index.tsx create mode 100644 playground/src/platform/pc/rtc/streamPlayer/index.module.scss create mode 100644 playground/src/platform/pc/rtc/streamPlayer/index.tsx create mode 100644 playground/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx delete mode 100644 playground/src/protobuf/SttMessage001.js delete mode 100644 playground/src/protobuf/SttMessage002.js delete mode 100644 playground/src/protobuf/SttMessage003.js diff --git a/playground/package.json b/playground/package.json index d67fddb7..85ecc3db 100644 --- a/playground/package.json +++ b/playground/package.json @@ -19,9 +19,11 @@ "@reduxjs/toolkit": "^2.2.3", "antd": "^5.15.3", "@ant-design/icons": "^5.3.7", - "agora-rtc-sdk-ng": "^4.21.0" + "agora-rtc-sdk-ng": "^4.21.0", + "react-colorful": "^5.6.1" }, "devDependencies": { + "@minko-fe/postcss-pxtoviewport": "^1.3.2", "typescript": "^5", "@types/node": "^20", "@types/react": "^18", @@ -34,5 +36,6 @@ "sass": "^1.77.5", "@svgr/webpack": "^8.1.0", "protobufjs-cli": "^1.1.2" - } + }, + "packageManager": "yarn@1.22.22+sha1.ac34549e6aa8e7ead463a7407e1c7390f61a6610" } diff --git a/playground/postcss.config.js b/playground/postcss.config.js index 90d9fffc..ea748d4d 100644 --- a/playground/postcss.config.js +++ b/playground/postcss.config.js @@ -1,5 +1,10 @@ module.exports = { plugins: { autoprefixer: {}, + "@minko-fe/postcss-pxtoviewport": { + viewportWidth: 375, + exclude: /node_modules/, + include: /\/src\/platform\/mobile\//, + } }, } diff --git a/playground/src/css/globals.css b/playground/src/app/global.css similarity index 74% rename from playground/src/css/globals.css rename to playground/src/app/global.css index 7d9fb8d2..daded34b 100644 --- a/playground/src/css/globals.css +++ b/playground/src/app/global.css @@ -8,9 +8,6 @@ html, body { background-color: #0F0F11; font-family: "PingFang SC"; - width: 100vw; - height: 100vh; - overflow: auto; } a { @@ -51,3 +48,18 @@ a { color: var(--Grey-300, #EAECF0) !important; } + +.ant-popover-inner { + /* width: 260px !important; */ + background: #1E2025 !important; +} + + +.ant-select-selection-placeholder { + color: var(--Grey-600, #667085) !important; +} + + +.ant-empty-description { + color: var(--Grey-600, #667085) !important; +} diff --git a/playground/src/app/home/index.module.scss b/playground/src/app/home/index.module.scss deleted file mode 100644 index 8b6f8011..00000000 --- a/playground/src/app/home/index.module.scss +++ /dev/null @@ -1,70 +0,0 @@ -.home { - display: flex; - flex-direction: column; - position: relative; - width: 100%; - height: 100%; - background-color: #0F0F11; - box-sizing: border-box; - - .content { - margin-top: 48px; - flex: 1 1 auto; - display: flex; - width: 100%; - padding: 18px 22px; - align-items: flex-start; - gap: 24px; - flex-shrink: 0; - box-sizing: border-box; - } - - .smallScreen { - left: 12px; - right: 12px; - top: 64px; - bottom: 16px; - position: fixed; - - .menuWrapper { - position: absolute; - left: 0; - right: 0; - top: 0; - height: 32px; - } - - .bodyWrapper { - position: absolute; - left: 0; - right: 0; - top: 48px; - bottom: 0; - overflow-y: hidden; - overflow-x: hidden; - - .item { - position: absolute; - left: 0; - right: 0; - top: 0; - bottom: 0; - overflow-y: auto; - - &::-webkit-scrollbar { - width: 4px - } - - &::-webkit-scrollbar-track { - background-color: transparent; - } - - &::-webkit-scrollbar-thumb { - background-color: #6B6B6B; - border-radius: 5px; - } - } - } - - } -} diff --git a/playground/src/app/home/page.tsx b/playground/src/app/home/page.tsx index e0437f4e..aa5adfec 100644 --- a/playground/src/app/home/page.tsx +++ b/playground/src/app/home/page.tsx @@ -1,84 +1,23 @@ "use client" -import { useMemo, useState, useRef, useEffect } from "react" -import dynamic from "next/dynamic"; -import Chat from "@/components/chat" -import Setting from "@/components/setting" import AuthInitializer from "@/components/authInitializer" -import Menu from "@/components/menu" -const Rtc = dynamic(() => import("@/components/rtc"), { +import { isMobile } from "@/common" +import dynamic from 'next/dynamic' + +const PCEntry = dynamic(() => import('@/platform/pc/entry'), { ssr: false, -}); -const Header = dynamic(() => import("@/components/header"), { +}) + +const MobileEntry = dynamic(() => import('@/platform/mobile/entry'), { ssr: false, -}); -import { useSmallScreen, useAppSelector } from "@/common" -import styles from "./index.module.scss" +}) export default function Home() { - const chatItems = useAppSelector(state => state.global.chatItems) - const wrapperRef = useRef(null) - const [activeMenu, setActiveMenu] = useState("Settings") - const { isSmallScreen } = useSmallScreen() - - useEffect(() => { - if (!wrapperRef.current) { - return - } - if (!isSmallScreen) { - return - } - wrapperRef.current.scrollTop = wrapperRef.current.scrollHeight - }, [isSmallScreen, chatItems]) - - const onMenuChange = (item: string) => { - setActiveMenu(item) - } return ( -
-
- {isSmallScreen ? -
-
- -
-
-
- -
-
- -
-
- -
-
-
- : -
- - - -
- } -
+ {isMobile() ? : } - ); } diff --git a/playground/src/app/index.module.scss b/playground/src/app/index.module.scss index 4c59dc50..78585596 100644 --- a/playground/src/app/index.module.scss +++ b/playground/src/app/index.module.scss @@ -14,7 +14,9 @@ $shadows-big: multiple-box-shadow(100); .login { - position: relative; + position: absolute; + left: 0; + top: 0; width: 100%; height: 100%; overflow: hidden; diff --git a/playground/src/app/layout.tsx b/playground/src/app/layout.tsx index 66b5991f..5ca9b722 100644 --- a/playground/src/app/layout.tsx +++ b/playground/src/app/layout.tsx @@ -1,8 +1,9 @@ import { ConfigProvider } from "antd" import { StoreProvider } from "@/store"; -import "@/css/globals.css"; import type { Metadata, Viewport } from "next"; +import './global.css' + export const metadata: Metadata = { title: "Ai Agent", diff --git a/playground/src/assets/color_picker.svg b/playground/src/assets/color_picker.svg new file mode 100644 index 00000000..fb9bb33e --- /dev/null +++ b/playground/src/assets/color_picker.svg @@ -0,0 +1,17 @@ + + + + + + + + + + diff --git a/playground/src/assets/info.svg b/playground/src/assets/info.svg new file mode 100644 index 00000000..8ca99511 --- /dev/null +++ b/playground/src/assets/info.svg @@ -0,0 +1,3 @@ + + + diff --git a/playground/src/assets/logo.svg b/playground/src/assets/logo.svg index b9921348..af99893a 100644 --- a/playground/src/assets/logo.svg +++ b/playground/src/assets/logo.svg @@ -1,26 +1,46 @@ - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - \ No newline at end of file + diff --git a/playground/src/assets/logo_small.svg b/playground/src/assets/logo_small.svg new file mode 100644 index 00000000..34e755bd --- /dev/null +++ b/playground/src/assets/logo_small.svg @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/playground/src/assets/pdf.svg b/playground/src/assets/pdf.svg new file mode 100644 index 00000000..dc67f4d5 --- /dev/null +++ b/playground/src/assets/pdf.svg @@ -0,0 +1,3 @@ + + + diff --git a/playground/src/assets/voice.svg b/playground/src/assets/voice.svg new file mode 100644 index 00000000..86a880b0 --- /dev/null +++ b/playground/src/assets/voice.svg @@ -0,0 +1,3 @@ + + + diff --git a/playground/src/common/constant.ts b/playground/src/common/constant.ts index 27cf634e..e520a195 100644 --- a/playground/src/common/constant.ts +++ b/playground/src/common/constant.ts @@ -1,4 +1,4 @@ -import { IOptions, ColorItem } from "@/types" +import { IOptions, ColorItem, LanguageOptionItem, VoiceOptionItem } from "@/types" export const REQUEST_URL = process.env.NEXT_PUBLIC_REQUEST_URL ?? "" export const GITHUB_URL = "https://github.com/rte-design/ASTRA.ai" @@ -9,7 +9,7 @@ export const DEFAULT_OPTIONS: IOptions = { userId: 0 } export const DESCRIPTION = "This is an AI voice assistant powered by ASTRA.ai framework, Agora, Azure and ChatGPT." -export const LANG_OPTIONS = [ +export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [ { label: "English", value: "en-US" @@ -19,7 +19,7 @@ export const LANG_OPTIONS = [ value: "zh-CN" } ] -export const VOICE_OPTIONS = [ +export const VOICE_OPTIONS: VoiceOptionItem[] = [ { label: "Male", value: "male" @@ -31,22 +31,22 @@ export const VOICE_OPTIONS = [ ] export const COLOR_LIST: ColorItem[] = [{ active: "#0888FF", - default: "#112941" + default: "#143354" }, { active: "#563FD8", - default: "#221C40" + default: "#2C2553" }, { active: "#18A957", - default: "#112A1E" + default: "#173526" }, { active: "#FFAB08", - default: "#392B13" + default: "#423115" }, { active: "#FD5C63", - default: "#3C2023" + default: "#462629" }, { active: "#E225B2", - default: "#371530" + default: "#481C3F" }] diff --git a/playground/src/common/hooks.ts b/playground/src/common/hooks.ts index b58dd303..9759fa29 100644 --- a/playground/src/common/hooks.ts +++ b/playground/src/common/hooks.ts @@ -69,10 +69,9 @@ export const useMultibandTrackVolume = ( return frequencyBands; }; -export const useAutoScroll = (ref: React.RefObject) => { +export const useAutoScroll = (ref: React.RefObject) => { const callback: MutationCallback = (mutationList, observer) => { - // console.log("[test] callback", mutationList, observer) mutationList.forEach((mutation) => { switch (mutation.type) { case "childList": @@ -99,10 +98,6 @@ export const useAutoScroll = (ref: React.RefObject) => { observer.disconnect(); }; }, [ref]); - - - - } export const useSmallScreen = () => { diff --git a/playground/src/common/request.ts b/playground/src/common/request.ts index 0100cd99..8329911d 100644 --- a/playground/src/common/request.ts +++ b/playground/src/common/request.ts @@ -71,6 +71,42 @@ export const apiStopService = async (channel: string) => { return resp } +export const apiGetDocumentList = async () => { + const url = `${REQUEST_URL}/vector/document/preset/list` + let resp: any = await fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }) + resp = (await resp.json()) || {} + if (resp.code !== "0") { + throw new Error(resp.msg) + } + return resp +} + +export const apiUpdateDocument = async (options: { channel: string, collection: string, fileName: string }) => { + const url = `${REQUEST_URL}/vector/document/update` + const { channel, collection, fileName } = options + const data = { + request_id: genUUID(), + channel_name: channel, + collection: collection, + file_name: fileName + } + let resp: any = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(data), + }) + resp = (await resp.json()) || {} + return resp +} + + // ping/pong export const apiPing = async (channel: string) => { const url = `${REQUEST_URL}/ping` diff --git a/playground/src/common/utils.ts b/playground/src/common/utils.ts index 4a9d9496..d2a144f9 100644 --- a/playground/src/common/utils.ts +++ b/playground/src/common/utils.ts @@ -52,3 +52,8 @@ export const genUUID = () => { return v.toString(16) }) } + + +export const isMobile = () => { + return /Mobile|iPhone|iPad|Android|Windows Phone/i.test(navigator.userAgent) +} diff --git a/playground/src/components/chat/chatItem/index.module.scss b/playground/src/components/chat/chatItem/index.module.scss deleted file mode 100644 index f1a17165..00000000 --- a/playground/src/components/chat/chatItem/index.module.scss +++ /dev/null @@ -1,62 +0,0 @@ -.chatItem { - display: flex; - justify-content: flex-start; - - .left { - flex: 0 0 auto; - display: flex; - width: 32px; - height: 32px; - padding: 10px; - flex-direction: column; - justify-content: center; - align-items: center; - gap: 10px; - border-radius: 200px; - background: var(--Grey-700, #475467); - - .text { - color: var(---white, #FFF); - text-align: center; - font-size: 14px; - font-weight: 500; - line-height: 150%; - } - } - - .right { - margin-left: 12px; - flex: 1 1 auto; - - .userName { - color: var(--Grey-600, #667085); - font-size: 12px; - font-weight: 400; - line-height: 18px; - } - - .userName.isAgent { - color: var(--theme-color, #667085) !important; - } - - .text { - margin-top: 6px; - color: var(--Grey-300, #EAECF0); - font-size: 14px; - font-weight: 400; - line-height: 21px; - padding-right: 4px; - white-space: pre-wrap; - display: inline; - } - - .text.isAgent { - color: var(--theme-color, #EAECF0) !important; - } - - } -} - -.chatItem+.chatItem { - margin-top: 10px; -} diff --git a/playground/src/components/chat/chatItem/index.tsx b/playground/src/components/chat/chatItem/index.tsx deleted file mode 100644 index e814039a..00000000 --- a/playground/src/components/chat/chatItem/index.tsx +++ /dev/null @@ -1,41 +0,0 @@ -import { IChatItem } from "@/types" -import styles from "./index.module.scss" -import { usePrevious } from "@/common" -import { use, useEffect, useMemo, useState } from "react" - -interface ChatItemProps { - data: IChatItem -} - - -let flag = false - -const ChatItem = (props: ChatItemProps) => { - const { data } = props - const { text, type } = data - - - const abUserName = useMemo(() => { - return type == "agent" ? "Ag" : "Yo" - }, [type]) - - const coUserName = useMemo(() => { - return type == "agent" ? "Agent" : "You" - }, [type]) - - - return
- - {abUserName} - - -
{coUserName}
-
- {text} -
- -
-} - - -export default ChatItem diff --git a/playground/src/components/chat/index.tsx b/playground/src/components/chat/index.tsx deleted file mode 100644 index 352fc108..00000000 --- a/playground/src/components/chat/index.tsx +++ /dev/null @@ -1,32 +0,0 @@ -"use client" - -import { useEffect, useRef } from "react" -import ChatItem from "./chatItem" -import { TranscriptionIcon } from "@/components/icons" -import { genRandomChatList, useSmallScreen, useAutoScroll, useAppSelector } from "@/common" -import styles from "./index.module.scss" - -const MOCK_CHAT_LIST = genRandomChatList(10) - -const Chat = () => { - const chatItems = useAppSelector(state => state.global.chatItems) - // const chatItems = MOCK_CHAT_LIST - const chatRef = useRef(null) - const { isSmallScreen } = useSmallScreen() - useAutoScroll(chatRef) - - return
-
- - Chat -
-
- {chatItems.map((item, index) => { - return - })} -
-
-} - - -export default Chat diff --git a/playground/src/components/customSelect/index.module.scss b/playground/src/components/customSelect/index.module.scss new file mode 100644 index 00000000..0649e994 --- /dev/null +++ b/playground/src/components/customSelect/index.module.scss @@ -0,0 +1,22 @@ +.selectWrapper { + position: relative; + + .prefixIconWrapper { + position: absolute; + z-index: 1; + width: 3rem; + height: 100%; + display: flex; + align-items: center; + justify-content: center; + } + + :global(.customSelect) { + width: 100%; + + :global(.ant-select-selector) { + padding-left: calc(3rem - 8px) !important; + } + } + +} diff --git a/playground/src/components/customSelect/index.tsx b/playground/src/components/customSelect/index.tsx new file mode 100644 index 00000000..8dd1b188 --- /dev/null +++ b/playground/src/components/customSelect/index.tsx @@ -0,0 +1,19 @@ +import { Select, SelectProps } from "antd" +import styles from "./index.module.scss" + +type CustomSelectProps = SelectProps & { + prefixIcon?: React.ReactNode; +} + +const CustomSelect = (props: CustomSelectProps) => { + + const { prefixIcon, className, ...rest } = props; + + return
+ {prefixIcon &&
{prefixIcon}
} + +
+} + + +export default CustomSelect diff --git a/playground/src/components/header/index.tsx b/playground/src/components/header/index.tsx deleted file mode 100644 index c8ed31f8..00000000 --- a/playground/src/components/header/index.tsx +++ /dev/null @@ -1,38 +0,0 @@ -"use client" - -import { useAppSelector, GITHUB_URL, useSmallScreen } from "@/common" -import Network from "./network" -import { GithubIcon, LogoIcon } from "@/components/icons" - -import styles from "./index.module.scss" -import { useMemo } from "react" - -const Header = () => { - const options = useAppSelector(state => state.global.options) - const { channel } = options - const { isSmallScreen } = useSmallScreen() - - const channelNameText = useMemo(() => { - return !isSmallScreen ? `Channel Name：${channel}` : channel - }, [isSmallScreen, channel]) - - const onClickGithub = () => { - if (typeof window !== "undefined") { - window.open(GITHUB_URL, "_blank") - } - } - - return
- - - - {channelNameText} - - - - -
-} - - -export default Header diff --git a/playground/src/components/icons/colorPicker/index.tsx b/playground/src/components/icons/colorPicker/index.tsx new file mode 100644 index 00000000..81efcb12 --- /dev/null +++ b/playground/src/components/icons/colorPicker/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import ColorPickerSvg from "@/assets/color_picker.svg" + +export const ColorPickerIcon = (props: IconProps) => { + return +} diff --git a/playground/src/components/icons/index.tsx b/playground/src/components/icons/index.tsx index deae1f21..e303674a 100644 --- a/playground/src/components/icons/index.tsx +++ b/playground/src/components/icons/index.tsx @@ -4,3 +4,7 @@ export * from "./network" export * from "./github" export * from "./transcription" export * from "./logo" +export * from "./info" +export * from "./colorPicker" +export * from "./voice" +export * from "./pdf" diff --git a/playground/src/components/icons/info/index.tsx b/playground/src/components/icons/info/index.tsx new file mode 100644 index 00000000..cf783be9 --- /dev/null +++ b/playground/src/components/icons/info/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import InfoSvg from "@/assets/info.svg" + +export const InfoIcon = (props: IconProps) => { + return +} diff --git a/playground/src/components/icons/logo/index.tsx b/playground/src/components/icons/logo/index.tsx index 53cb20fc..f86d5246 100644 --- a/playground/src/components/icons/logo/index.tsx +++ b/playground/src/components/icons/logo/index.tsx @@ -1,6 +1,8 @@ import { IconProps } from "../types" import LogoSvg from "@/assets/logo.svg" +import SmallLogoSvg from "@/assets/logo_small.svg" export const LogoIcon = (props: IconProps) => { - return + const { size = "default" } = props + return size == "small" ? : } diff --git a/playground/src/components/icons/pdf/index.tsx b/playground/src/components/icons/pdf/index.tsx new file mode 100644 index 00000000..83de8b2d --- /dev/null +++ b/playground/src/components/icons/pdf/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import PdfSvg from "@/assets/pdf.svg" + +export const PdfIcon = (props: IconProps) => { + return +} diff --git a/playground/src/components/icons/types.ts b/playground/src/components/icons/types.ts index 61bd05b2..c37e8133 100644 --- a/playground/src/components/icons/types.ts +++ b/playground/src/components/icons/types.ts @@ -3,6 +3,8 @@ export interface IconProps { height?: number color?: string viewBox?: string + size?: "small" | "default" // style?: React.CSSProperties transform?: string + onClick?: () => void } diff --git a/playground/src/components/icons/voice/index.tsx b/playground/src/components/icons/voice/index.tsx new file mode 100644 index 00000000..87164cea --- /dev/null +++ b/playground/src/components/icons/voice/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import VoiceSvg from "@/assets/voice.svg" + +export const VoiceIcon = (props: IconProps) => { + return +} diff --git a/playground/src/components/loginCard/index.module.scss b/playground/src/components/loginCard/index.module.scss index 52ea07e6..966ebc20 100644 --- a/playground/src/components/loginCard/index.module.scss +++ b/playground/src/components/loginCard/index.module.scss @@ -47,7 +47,7 @@ gap: 12px; .text { - margin-top: 12px; + margin-top: 8px; color: var(--Grey-300, #EAECF0); text-align: center; font-size: 18px; diff --git a/playground/src/components/loginCard/index.tsx b/playground/src/components/loginCard/index.tsx index f672e79c..a161e912 100644 --- a/playground/src/components/loginCard/index.tsx +++ b/playground/src/components/loginCard/index.tsx @@ -1,14 +1,17 @@ "use client" -import { version } from "../../../package.json" +import packageData from "../../../package.json" import { useRouter } from 'next/navigation' import { message } from "antd" -import { ChangeEvent, InputHTMLAttributes, useState } from "react" -import { GithubIcon,LogoIcon } from "../icons" +import { useState } from "react" +import { GithubIcon, LogoIcon } from "../icons" import { GITHUB_URL, getRandomUserId, useAppDispatch, getRandomChannel } from "@/common" import { setOptions } from "@/store/reducers/global" import styles from "./index.module.scss" + +const { version } = packageData + const LoginCard = () => { const dispatch = useAppDispatch() const router = useRouter() @@ -52,8 +55,8 @@ const LoginCard = () => {
- - ASTRA.ai Agents Playground + + Agents Playground
@@ -66,6 +69,9 @@ const LoginCard = () => {
Version {version}
+ + + return } export default LoginCard diff --git a/playground/src/components/menu/index.module.scss b/playground/src/components/menu/index.module.scss deleted file mode 100644 index bd6f2c94..00000000 --- a/playground/src/components/menu/index.module.scss +++ /dev/null @@ -1,24 +0,0 @@ -.menu { - width: 100%; - background: #181A1E; - border: 1px solid #272A2F; - box-sizing: border-box; - border-radius: 4px; - - .menuItem { - display: inline-block; - height: 32px; - line-height: 32px; - padding: 0 15px; - color: #667085; - background: #181A1E; - cursor: pointer; - border-right: 1px solid #272A2F; - box-sizing: border-box; - } - - .active { - color: #EAECF0; - background: #0F0F11; - } -} diff --git a/playground/src/components/menu/index.tsx b/playground/src/components/menu/index.tsx deleted file mode 100644 index 524fdde8..00000000 --- a/playground/src/components/menu/index.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import { useState } from "react" -import styles from "./index.module.scss" - -interface MenuProps { - onChange: (text: string) => void -} - -interface MenuItem { - text: string - active: boolean -} - -const DEFAULT_MENU_LIST: MenuItem[] = [ - { - text: "Settings", - active: true - }, { - text: "Chat", - active: false - }, { - text: "Agent", - active: false - }] - -const Menu = (props: MenuProps) => { - const { onChange } = props - const [menuList, setMenuList] = useState(DEFAULT_MENU_LIST) - - const onClickItem = (index: number) => { - if (menuList[index].active) { - return - } - const newMenuList = menuList.map((item, i) => { - return { - ...item, - active: i == index - } - }) - setMenuList(newMenuList) - onChange(menuList[index].text) - } - - return
- {menuList.map((item, index) => { - return onClickItem(index)}>{item.text} - })} -
-} - -export default Menu diff --git a/playground/src/components/pdfSelect/index.module.scss b/playground/src/components/pdfSelect/index.module.scss new file mode 100644 index 00000000..adb93280 --- /dev/null +++ b/playground/src/components/pdfSelect/index.module.scss @@ -0,0 +1,8 @@ +// .pdfSelect { + // min-width: 200px; + // max-width: 300px; + // } +.dropdownRender { + display: flex; + justify-content: flex-end; +} diff --git a/playground/src/components/pdfSelect/index.tsx b/playground/src/components/pdfSelect/index.tsx new file mode 100644 index 00000000..31544e60 --- /dev/null +++ b/playground/src/components/pdfSelect/index.tsx @@ -0,0 +1,72 @@ +import { ReactElement, useState } from "react" +import { PdfIcon } from "@/components/icons" +import CustomSelect from "@/components/customSelect" +import { Divider, message } from 'antd'; +import { useEffect } from 'react'; +import { apiGetDocumentList, apiUpdateDocument, useAppSelector } from "@/common" +import PdfUpload from "./upload" +import { OptionType, IPdfData } from "@/types" + +import styles from "./index.module.scss" + +const PdfSelect = () => { + const options = useAppSelector(state => state.global.options) + const { channel } = options + const [pdfOptions, setPdfOptions] = useState([]) + + + useEffect(() => { + getPDFOptions() + }, []) + + + const getPDFOptions = async () => { + const res = await apiGetDocumentList() + setPdfOptions(res.data.map((item: any) => { + return { + value: item.collection, + label: item.file_name + } + })) + } + + const onUploadSuccess = (data: IPdfData) => { + setPdfOptions([...pdfOptions, { + value: data.collection, + label: data.fileName + }]) + } + + const pdfDropdownRender = (menu: ReactElement) => { + return <> + {menu} + +
+ +
+ + } + + + const onSelectPdf = async (val: string) => { + const item = pdfOptions.find(item => item.value === val) + if (!item) { + return message.error("Please select a PDF file") + } + await apiUpdateDocument({ + collection: val, + fileName: item.label, + channel + }) + } + + + return } + onChange={onSelectPdf} + options={pdfOptions} + dropdownRender={pdfDropdownRender} + className={styles.pdfSelect} placeholder="Select a PDF file"> +} + +export default PdfSelect diff --git a/playground/src/components/pdfSelect/upload/index.module.scss b/playground/src/components/pdfSelect/upload/index.module.scss new file mode 100644 index 00000000..fd559b5e --- /dev/null +++ b/playground/src/components/pdfSelect/upload/index.module.scss @@ -0,0 +1,7 @@ +.btn { + color: var(--theme-color, #EAECF0); + + &:hover { + color: var(--theme-color, #EAECF0) !important; + } +} diff --git a/playground/src/components/pdfSelect/upload/index.tsx b/playground/src/components/pdfSelect/upload/index.tsx new file mode 100644 index 00000000..043d054b --- /dev/null +++ b/playground/src/components/pdfSelect/upload/index.tsx @@ -0,0 +1,74 @@ +import { Select, Button, message, Upload, UploadProps } from "antd" +import { useState } from "react" +import { PlusOutlined, LoadingOutlined } from '@ant-design/icons'; +import { REQUEST_URL, useAppSelector, genUUID } from "@/common" +import { IPdfData } from "@/types" + +import styles from "./index.module.scss" + +interface PdfSelectProps { + onSuccess?: (data: IPdfData) => void +} + +const PdfUpload = (props: PdfSelectProps) => { + const { onSuccess } = props + const agentConnected = useAppSelector(state => state.global.agentConnected) + const options = useAppSelector(state => state.global.options) + const { channel, userId } = options + + const [uploading, setUploading] = useState(false) + + const uploadProps: UploadProps = { + accept: "application/pdf", + maxCount: 1, + showUploadList: false, + action: `${REQUEST_URL}/vector/document/upload`, + data: { + channel_name: channel, + uid: String(userId), + request_id: genUUID() + }, + onChange: (info) => { + const { file } = info + const { status, name } = file + if (status == "uploading") { + setUploading(true) + } else if (status == 'done') { + setUploading(false) + const { response } = file + if (response.code == "0") { + message.success(`Upload ${name} success`) + const { collection, file_name } = response.data + onSuccess && onSuccess({ + fileName: file_name, + collection + }) + } else { + message.error(response.msg) + } + } else if (status == 'error') { + setUploading(false) + message.error(`Upload ${name} failed`) + } + } + } + + const onClickUploadPDF = (e: any) => { + if (!agentConnected) { + message.error("Please connect to agent first") + e.stopPropagation() + } + } + + + return + + +} + + +export default PdfUpload diff --git a/playground/src/components/setting/Info/index.tsx b/playground/src/components/setting/Info/index.tsx deleted file mode 100644 index cdc20dfa..00000000 --- a/playground/src/components/setting/Info/index.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import { useAppSelector } from "@/common" - -import styles from "./index.module.scss" - -const Info = () => { - const options = useAppSelector(state => state.global.options) - const { channel, userId } = options - - return
-
INFO
-
- Room - {channel} -
-
- Participant - {userId} -
-
-} - -export default Info diff --git a/playground/src/components/setting/index.module.scss b/playground/src/components/setting/index.module.scss deleted file mode 100644 index 314994c3..00000000 --- a/playground/src/components/setting/index.module.scss +++ /dev/null @@ -1,100 +0,0 @@ -.setting { - flex: 0 0 260px; - display: flex; - flex-direction: column; - align-items: flex-start; - flex-shrink: 0; - align-self: stretch; - border-radius: 8px; - border: 1px solid #272A2F; - background: #181A1D; - - - .description { - display: flex; - padding: 24px 16px; - flex-direction: column; - align-items: flex-start; - gap: 12px; - align-self: stretch; - border-bottom: 1px solid #272A2F; - - .title { - color: var(--Grey-300, #EAECF0); - font-size: 14px; - font-weight: 600; - line-height: 150%; - letter-spacing: 0.449px; - } - - .text { - color: var(--Grey-600, #667085); - font-size: 14px; - font-weight: 400; - line-height: 150%; - } - - .btnConnect { - display: flex; - padding: 8px 14px; - justify-content: center; - align-items: center; - gap: 8px; - align-self: stretch; - border-radius: 6px; - background: var(--theme-color, #0888FF); - box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05); - cursor: pointer; - user-select: none; - caret-color: transparent; - - .btnText { - color: var(---White, #FFF); - font-size: 14px; - font-weight: 500; - line-height: 20px; - } - - .btnText.disconnect { - color: var(--Error-400-T, #E95C7B); - } - - .loading { - margin-left: 4px; - } - } - - - .btnConnect.disconnect { - background: #181A1D; - border: 1px solid var(--Error-400-T, #E95C7B); - } - - - } - - - .selectWrapper { - display: flex; - padding: 24px 16px; - flex-direction: column; - align-items: flex-start; - gap: 12px; - align-self: stretch; - border: 1px solid #272A2F; - - .title { - color: var(--Grey-300, #EAECF0); - font-size: 14px; - font-weight: 600; - line-height: 150%; - letter-spacing: 0.449px; - } - - .select { - width: 100%; - } - } - - -} diff --git a/playground/src/platform/mobile/chat/chatItem/index.module.scss b/playground/src/platform/mobile/chat/chatItem/index.module.scss new file mode 100644 index 00000000..27057120 --- /dev/null +++ b/playground/src/platform/mobile/chat/chatItem/index.module.scss @@ -0,0 +1,86 @@ +.agentChatItem { + width: 100%; + display: flex; + justify-content: flex-start; + + .left { + flex: 0 0 auto; + display: flex; + width: 32px; + height: 32px; + padding: 10px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + border-radius: 200px; + background: var(--Grey-700, #475467); + + .userName { + color: var(---white, #FFF); + text-align: center; + font-size: 14px; + font-weight: 500; + line-height: 150%; + } + } + + .right { + margin-left: 12px; + + .userName { + font-size: 14px; + font-weight: 500; + line-height: 20px; + color: var(--theme-color, #667085) !important; + } + + + .agent { + color: var(--theme-color, #EAECF0) !important; + } + + } +} + +.userChatItem { + width: 100%; + display: flex; + flex-direction: column; + justify-content: flex-end; + align-items: flex-end; + + .userName { + text-align: right; + color: var(--Grey-600, #667085); + font-weight: 500; + line-height: 20px; + } + + + +} + + +.chatItem { + .text { + margin-top: 6px; + color: #FFF; + display: flex; + padding: 8px 14px; + flex-direction: column; + justify-content: left; + font-size: 14px; + font-weight: 400; + line-height: 21px; + white-space: pre-wrap; + border-radius: 0px 8px 8px 8px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} + +.chatItem+.chatItem { + margin-top: 14px; +} diff --git a/playground/src/platform/mobile/chat/chatItem/index.tsx b/playground/src/platform/mobile/chat/chatItem/index.tsx new file mode 100644 index 00000000..bde3350c --- /dev/null +++ b/playground/src/platform/mobile/chat/chatItem/index.tsx @@ -0,0 +1,50 @@ +import { IChatItem } from "@/types" +import styles from "./index.module.scss" + +interface ChatItemProps { + data: IChatItem +} + + +const AgentChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + + return
+ + Ag + + +
Agent
+
+ {text} +
+ +
+} + +const UserChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + return
+
You
+
{text}
+
+} + + +const ChatItem = (props: ChatItemProps) => { + const { data } = props + + + return ( + data.type === "agent" ? : + ); + + +} + + +export default ChatItem diff --git a/playground/src/components/chat/index.module.scss b/playground/src/platform/mobile/chat/index.module.scss similarity index 76% rename from playground/src/components/chat/index.module.scss rename to playground/src/platform/mobile/chat/index.module.scss index 50fdb1e5..8fa6d901 100644 --- a/playground/src/components/chat/index.module.scss +++ b/playground/src/platform/mobile/chat/index.module.scss @@ -1,23 +1,18 @@ .chat { flex: 1 1 auto; - min-width: 200px; display: flex; flex-direction: column; align-items: flex-start; align-self: stretch; - border-radius: 8px; - border: 1px solid #272A2F; background: #181A1D; overflow: hidden; .header { - display: flex; - height: 40px; - padding: 0px 16px; align-items: center; align-self: stretch; border-bottom: 1px solid #272A2F; + .text { margin-left: 4px; color: var(--Grey-300, #EAECF0); @@ -27,15 +22,22 @@ line-height: 40px; letter-spacing: 0.449px; } + + .languageSelect { + width: 100%; + } + + + + } .content { + margin-top: 16px; display: flex; - padding: 12px 24px; flex-direction: column; align-items: flex-start; gap: 10px; - flex: 1 0 500px; align-self: stretch; overflow-y: auto; @@ -55,9 +57,19 @@ } - .content.small { - flex: 1 1 auto; - min-height: 400px; - } +} + + +.dropdownRender { + display: flex; + justify-content: flex-end; + + .btn { + color: var(--theme-color, #EAECF0); + + &:hover { + color: var(--theme-color, #EAECF0) !important; + } + } } diff --git a/playground/src/platform/mobile/chat/index.tsx b/playground/src/platform/mobile/chat/index.tsx new file mode 100644 index 00000000..e18c0558 --- /dev/null +++ b/playground/src/platform/mobile/chat/index.tsx @@ -0,0 +1,64 @@ +import { ReactElement, useEffect, useContext, useState } from "react" +import ChatItem from "./chatItem" +import { IChatItem } from "@/types" +import { useAppDispatch, useAutoScroll, LANGUAGE_OPTIONS, useAppSelector } from "@/common" +import { setLanguage } from "@/store/reducers/global" +import { Select, } from 'antd'; +import { MenuContext } from "../menu/context" +import PdfSelect from "@/components/pdfSelect" + +import styles from "./index.module.scss" + + +const Chat = () => { + const chatItems = useAppSelector(state => state.global.chatItems) + const language = useAppSelector(state => state.global.language) + const agentConnected = useAppSelector(state => state.global.agentConnected) + const dispatch = useAppDispatch() + // genRandomChatList + // const [chatItems, setChatItems] = useState([]) + const context = useContext(MenuContext); + + if (!context) { + throw new Error("MenuContext is not found") + } + + const { scrollToBottom } = context; + + + useEffect(() => { + scrollToBottom() + }, [chatItems, scrollToBottom]) + + + + const onLanguageChange = (val: any) => { + dispatch(setLanguage(val)) + } + + + + return
+
+
+ +
+
+ +
+
+
+ {chatItems.map((item, index) => { + return + })} +
+
+} + + +export default Chat diff --git a/playground/src/platform/mobile/description/index.module.scss b/playground/src/platform/mobile/description/index.module.scss new file mode 100644 index 00000000..7305f5a7 --- /dev/null +++ b/playground/src/platform/mobile/description/index.module.scss @@ -0,0 +1,71 @@ +.description { + position: relative; + display: flex; + padding: 12px 16px; + height: 60px; + align-items: center; + gap: 12px; + align-self: stretch; + border-bottom: 1px solid #272A2F; + background: #181A1D; + box-sizing: border-box; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-style: normal; + font-weight: 600; + flex: 1 1 auto; + /* 21px */ + letter-spacing: 0.449px; + } + + .text { + margin-left: 12px; + flex: 1 1 auto; + color: var(--Grey-600, #667085); + font-size: 14px; + font-style: normal; + font-weight: 400; + } + + + .btnConnect { + width: 150px; + display: flex; + padding: 8px 14px; + justify-content: center; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 6px; + background: var(--theme-color, #0888FF); + box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05); + cursor: pointer; + user-select: none; + caret-color: transparent; + box-sizing: border-box; + + .btnText { + color: var(---White, #FFF); + font-size: 14px; + font-weight: 500; + line-height: 20px; + } + + .btnText.disconnect { + color: var(--Error-400-T, #E95C7B); + } + + .loading { + margin-left: 4px; + } + } + + + .btnConnect.disconnect { + background: #181A1D; + border: 1px solid var(--Error-400-T, #E95C7B); + } + +} diff --git a/playground/src/components/setting/index.tsx b/playground/src/platform/mobile/description/index.tsx similarity index 50% rename from playground/src/components/setting/index.tsx rename to playground/src/platform/mobile/description/index.tsx index d48feb30..505bb2d7 100644 --- a/playground/src/components/setting/index.tsx +++ b/playground/src/platform/mobile/description/index.tsx @@ -1,29 +1,22 @@ -"use client" - import { setAgentConnected } from "@/store/reducers/global" import { - DESCRIPTION, useAppDispatch, useAppSelector, apiPing, - LANG_OPTIONS, VOICE_OPTIONS, apiStartService, apiStopService + DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, + apiStartService, apiStopService, REQUEST_URL } from "@/common" -import Info from "./Info" -import Status from "./status" -import { Select, Button, message } from "antd" -import StyleSelect from "./themeSelect" +import { message } from "antd" import { useEffect, useState } from "react" -import { LoadingOutlined } from "@ant-design/icons" +import { LoadingOutlined, } from "@ant-design/icons" import styles from "./index.module.scss" - - let intervalId: any -const Setting = () => { +const Description = () => { const dispatch = useAppDispatch() const agentConnected = useAppSelector(state => state.global.agentConnected) const channel = useAppSelector(state => state.global.options.channel) const userId = useAppSelector(state => state.global.options.userId) - const [lang, setLang] = useState("en-US") - const [voice, setVoice] = useState("male") + const language = useAppSelector(state => state.global.language) + const voiceType = useAppSelector(state => state.global.voiceType) const [loading, setLoading] = useState(false) useEffect(() => { @@ -54,8 +47,8 @@ const Setting = () => { const res = await apiStartService({ channel, userId, - language: lang, - voiceType: voice + language, + voiceType }) const { code, msg } = res || {} if (code != 0) { @@ -90,40 +83,16 @@ const Setting = () => { } } - - return
- {/* description */} -
-
DESCRIPTION
-
{DESCRIPTION}
-
- - {!agentConnected ? "Connect" : "Disconnect"} - {loading ? : null} - -
-
- {/* info */} - - {/* status */} - - {/* select */} -
-
LANGUAGE
- -
-
-
Voice
- -
- {/* style */} - -
+ return
+ Description + + + {!agentConnected ? "Connect" : "Disconnect"} + {loading ? : null} + + +
} -export default Setting +export default Description diff --git a/playground/src/platform/mobile/entry/index.module.scss b/playground/src/platform/mobile/entry/index.module.scss new file mode 100644 index 00000000..41322c12 --- /dev/null +++ b/playground/src/platform/mobile/entry/index.module.scss @@ -0,0 +1,18 @@ +.entry { + position: relative; + height: 100%; + box-sizing: border-box; + + .content { + position: relative; + padding: 16px; + box-sizing: border-box; + + + .body { + margin-top: 16px; + display: flex; + gap: 24px; + } + } +} diff --git a/playground/src/platform/mobile/entry/index.tsx b/playground/src/platform/mobile/entry/index.tsx new file mode 100644 index 00000000..c5f51d5c --- /dev/null +++ b/playground/src/platform/mobile/entry/index.tsx @@ -0,0 +1,30 @@ +import Chat from "../chat" +import Description from "../description" +import Rtc from "../rtc" +import Header from "../header" +import Menu, { IMenuData } from "../menu" +import styles from "./index.module.scss" + + +const MenuData: IMenuData[] = [{ + name: "Agent", + component: , +}, { + name: "Chat", + component: , +}] + + +const MobileEntry = () => { + + return
+
+ +
+ +
+
+} + + +export default MobileEntry diff --git a/playground/src/platform/mobile/header/index.module.scss b/playground/src/platform/mobile/header/index.module.scss new file mode 100644 index 00000000..96afeb9c --- /dev/null +++ b/playground/src/platform/mobile/header/index.module.scss @@ -0,0 +1,51 @@ +.header { + display: flex; + width: 100%; + height: 48px; + padding: 16px; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #24262A; + background: #1E2024; + box-shadow: 0px 12px 16px -4px rgba(8, 15, 52, 0.06), 0px 4px 6px -2px rgba(8, 15, 52, 0.03); + box-sizing: border-box; + z-index: 999; + + .logoWrapper { + display: flex; + align-items: center; + + .text { + margin-left: 8px; + color: var(---white, #FFF); + text-align: right; + font-family: Inter; + font-size: 16px; + font-weight: 500; + } + } + + .content { + padding-left: 12px; + display: flex; + align-items: center; + justify-content: flex-start; + height: 48px; + flex: 1 1 auto; + color: var(--Grey-300, #EAECF0); + font-size: 16px; + font-weight: 500; + line-height: 48px; + letter-spacing: 0.449px; + text-align: center; + + .text { + margin-left: 4px; + font-size: 12px; + } + } + + .githubWrapper { + margin-right: 12px; + } +} diff --git a/playground/src/platform/mobile/header/index.tsx b/playground/src/platform/mobile/header/index.tsx new file mode 100644 index 00000000..50d21ddf --- /dev/null +++ b/playground/src/platform/mobile/header/index.tsx @@ -0,0 +1,46 @@ +"use client" + +import { useAppSelector, GITHUB_URL, useSmallScreen } from "@/common" +import Network from "./network" +import InfoPopover from "./infoPopover" +import StylePopover from "./stylePopover" +import { GithubIcon, LogoIcon, InfoIcon, ColorPickerIcon } from "@/components/icons" + +import styles from "./index.module.scss" + +const Header = () => { + const themeColor = useAppSelector(state => state.global.themeColor) + const options = useAppSelector(state => state.global.options) + const { channel } = options + + + const onClickGithub = () => { + if (typeof window !== "undefined") { + window.open(GITHUB_URL, "_blank") + } + } + + + + return
+ + + + + + + Channel Name: {channel} + + + + + + + + + +
+} + + +export default Header diff --git a/playground/src/components/setting/Info/index.module.scss b/playground/src/platform/mobile/header/infoPopover/index.module.scss similarity index 84% rename from playground/src/components/setting/Info/index.module.scss rename to playground/src/platform/mobile/header/infoPopover/index.module.scss index b5bad7a2..cd3f72f8 100644 --- a/playground/src/components/setting/Info/index.module.scss +++ b/playground/src/platform/mobile/header/infoPopover/index.module.scss @@ -1,11 +1,10 @@ .info { display: flex; - padding: 24px 16px; + padding: 12px 16px; flex-direction: column; align-items: flex-start; - gap: 12px; + gap: 8px; align-self: stretch; - border-bottom: 1px solid #272A2F; .title { color: var(--Grey-300, #EAECF0); @@ -35,4 +34,10 @@ line-height: 150%; } } + + .slider { + height: 1px; + width: 100%; + background-color: #0D0F12; + } } diff --git a/playground/src/components/setting/status/index.tsx b/playground/src/platform/mobile/header/infoPopover/index.tsx similarity index 51% rename from playground/src/components/setting/status/index.tsx rename to playground/src/platform/mobile/header/infoPopover/index.tsx index ed1418ea..cd451418 100644 --- a/playground/src/components/setting/status/index.tsx +++ b/playground/src/platform/mobile/header/infoPopover/index.tsx @@ -1,8 +1,19 @@ import { useMemo } from "react" import { useAppSelector } from "@/common" +import { Popover } from 'antd'; + + import styles from "./index.module.scss" -const Status = () => { +interface InfoPopoverProps { + children?: React.ReactNode +} + +const InfoPopover = (props: InfoPopoverProps) => { + const { children } = props + const options = useAppSelector(state => state.global.options) + const { channel, userId } = options + const roomConnected = useAppSelector(state => state.global.roomConnected) const agentConnected = useAppSelector(state => state.global.agentConnected) @@ -14,7 +25,19 @@ const Status = () => { return agentConnected ? "TRUE" : "FALSE" }, [agentConnected]) - return
+ + + const content =
+
INFO
+
+ Room + {channel} +
+
+ Participant + {userId} +
+
STATUS
Room connected
@@ -24,8 +47,11 @@ const Status = () => {
Agent connected
{agentConnectedText}
-
-} + -export default Status + return {children} + +} + +export default InfoPopover diff --git a/playground/src/components/header/network/index.module.scss b/playground/src/platform/mobile/header/network/index.module.scss similarity index 100% rename from playground/src/components/header/network/index.module.scss rename to playground/src/platform/mobile/header/network/index.module.scss diff --git a/playground/src/components/header/network/index.tsx b/playground/src/platform/mobile/header/network/index.tsx similarity index 78% rename from playground/src/components/header/network/index.tsx rename to playground/src/platform/mobile/header/network/index.tsx index 7e7c6501..92b4e33b 100644 --- a/playground/src/components/header/network/index.tsx +++ b/playground/src/platform/mobile/header/network/index.tsx @@ -1,11 +1,18 @@ "use client"; +import React from "react"; import { rtcManager } from "@/manager" import { NetworkQuality } from "agora-rtc-sdk-ng" import { useEffect, useState } from "react" import { NetworkIcon } from "@/components/icons" -const NetWork = () => { +interface NetworkProps { + style?: React.CSSProperties +} + +const NetWork = (props: NetworkProps) => { + const { style } = props + const [networkQuality, setNetworkQuality] = useState() useEffect(() => { @@ -21,7 +28,7 @@ const NetWork = () => { } return ( - + ) diff --git a/playground/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss b/playground/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss new file mode 100644 index 00000000..405e7781 --- /dev/null +++ b/playground/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss @@ -0,0 +1,24 @@ +.colorPicker { + height: 24px; + display: flex; + align-items: center; + + :global(.react-colorful) { + width: 220px; + height: 8px; + } + + :global(.react-colorful__saturation) { + display: none; + } + + :global(.react-colorful__hue) { + border-radius: 8px !important; + height: 8px; + } + + :global(.react-colorful__pointer) { + width: 24px; + height: 24px; + } +} diff --git a/playground/src/platform/mobile/header/stylePopover/colorPicker/index.tsx b/playground/src/platform/mobile/header/stylePopover/colorPicker/index.tsx new file mode 100644 index 00000000..28163d77 --- /dev/null +++ b/playground/src/platform/mobile/header/stylePopover/colorPicker/index.tsx @@ -0,0 +1,22 @@ +"use client" + +import { HexColorPicker } from "react-colorful"; +import { useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import styles from "./index.module.scss"; + +const ColorPicker = () => { + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + const onColorChange = (color: string) => { + console.log(color); + dispatch(setThemeColor(color)) + }; + + return
+ +
+}; + +export default ColorPicker; diff --git a/playground/src/components/setting/themeSelect/index.module.scss b/playground/src/platform/mobile/header/stylePopover/index.module.scss similarity index 92% rename from playground/src/components/setting/themeSelect/index.module.scss rename to playground/src/platform/mobile/header/stylePopover/index.module.scss index 8ecf6d43..defdcc12 100644 --- a/playground/src/components/setting/themeSelect/index.module.scss +++ b/playground/src/platform/mobile/header/stylePopover/index.module.scss @@ -1,11 +1,12 @@ -.style { +.info { + padding: 12px 16px; display: flex; - padding: 24px 16px; flex-direction: column; align-items: flex-start; - gap: 12px; + gap: 16px; align-self: stretch; + .title { color: var(--Grey-300, #EAECF0); font-size: 14px; @@ -16,7 +17,7 @@ .color { font-size: 0; - white-space:nowrap; + white-space: nowrap; .item { position: relative; @@ -47,5 +48,4 @@ } - } diff --git a/playground/src/components/setting/themeSelect/index.tsx b/playground/src/platform/mobile/header/stylePopover/index.tsx similarity index 61% rename from playground/src/components/setting/themeSelect/index.tsx rename to playground/src/platform/mobile/header/stylePopover/index.tsx index 58580cd9..f8508323 100644 --- a/playground/src/components/setting/themeSelect/index.tsx +++ b/playground/src/platform/mobile/header/stylePopover/index.tsx @@ -1,11 +1,21 @@ -import { useState } from "react" -import { COLOR_LIST, useAppSelector,useAppDispatch } from "@/common" -import styles from "./index.module.scss" +import { useMemo } from "react" +import { COLOR_LIST, useAppSelector, useAppDispatch } from "@/common" import { setThemeColor } from "@/store/reducers/global" +import ColorPicker from "./colorPicker" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface StylePopoverProps { + children?: React.ReactNode +} -const ThemeSelect = () => { +const StylePopover = (props: StylePopoverProps) => { + const { children } = props const dispatch = useAppDispatch() - const themeColor = useAppSelector((state) => state.global.themeColor) + const themeColor = useAppSelector(state => state.global.themeColor) + const onClickColor = (index: number) => { const target = COLOR_LIST[index] @@ -14,7 +24,7 @@ const ThemeSelect = () => { } } - return
+ const content =
STYLE
{ @@ -33,8 +43,12 @@ const ThemeSelect = () => { }) }
+
-} -export default ThemeSelect + return {children} + +} + +export default StylePopover diff --git a/playground/src/platform/mobile/menu/context.ts b/playground/src/platform/mobile/menu/context.ts new file mode 100644 index 00000000..41c52911 --- /dev/null +++ b/playground/src/platform/mobile/menu/context.ts @@ -0,0 +1,9 @@ +import { createContext } from "react" + +export interface MenuContextType { + scrollToBottom: () => void; +} + +export const MenuContext = createContext({ + scrollToBottom: () => { } +}); diff --git a/playground/src/platform/mobile/menu/index.module.scss b/playground/src/platform/mobile/menu/index.module.scss new file mode 100644 index 00000000..58b1b3fe --- /dev/null +++ b/playground/src/platform/mobile/menu/index.module.scss @@ -0,0 +1,69 @@ +.menu { + width: 100%; + border: 1px solid #272A2F; + border-radius: 4px; + background: #0F0F11; + overflow: hidden; + box-sizing: border-box; + + .header { + height: 40px; + overflow: hidden; + border-bottom: 1px solid #272A2F; + box-sizing: border-box; + + .menuItem { + height: 40px; + padding: 0 16px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 40px; + letter-spacing: 0.449px; + display: inline-block; + color: #667085; + background: #181A1E; + cursor: pointer; + border-right: 1px solid #272A2F; + box-sizing: border-box; + overflow: hidden; + background: #0F0F11; + } + + .active { + color: #EAECF0; + background: #181A1D; + } + } + + + .content { + position: relative; + background: #181A1D; + // header 48px + // description 60px + // paddingTop 16px 16px + // menu header 40px + height: calc(100vh - 48px - 60px - 32px - 40px - 2px); + overflow: hidden; + box-sizing: border-box; + + .item { + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; + padding: 16px; + z-index: -1; + overflow: auto; + visibility: hidden; + box-sizing: border-box; + } + + .active { + z-index: 1; + visibility: visible; + } + } +} diff --git a/playground/src/platform/mobile/menu/index.tsx b/playground/src/platform/mobile/menu/index.tsx new file mode 100644 index 00000000..2e20de78 --- /dev/null +++ b/playground/src/platform/mobile/menu/index.tsx @@ -0,0 +1,76 @@ +"use client" + +import { ReactElement, useEffect, useState, useRef, useMemo, useCallback } from "react" +import { useAutoScroll } from "@/common" +import { MenuContext } from "./context" +import styles from "./index.module.scss" + +export interface IMenuData { + name: string, + component: ReactElement +} + +export interface IMenuContentComponentPros { + scrollToBottom: () => void +} + +interface MenuProps { + data: IMenuData[] +} + + +const Menu = (props: MenuProps) => { + const { data } = props + const [activeIndex, setActiveIndex] = useState(0) + const contentRefList = useRef<(HTMLDivElement | null)[]>([]) + + const onClickItem = (index: number) => { + setActiveIndex(index) + } + + useEffect(() => { + scrollToTop() + }, [activeIndex]) + + const scrollToBottom = useCallback(() => { + const current = contentRefList.current?.[activeIndex] + if (current) { + current.scrollTop = current.scrollHeight + } + }, [contentRefList, activeIndex]) + + const scrollToTop = useCallback(() => { + const current = contentRefList.current?.[activeIndex] + if (current) { + current.scrollTop = 0 + } + }, [contentRefList, activeIndex]) + + + return
+
+ {data.map((item, index) => { + return onClickItem(index)}>{item.name} + })} +
+
+ + {data.map((item, index) => { + return
{ + contentRefList.current[index] = el; + }} + className={`${styles.item} ${index == activeIndex ? styles.active : ''}`}> + {item.component} +
+ })} + +
+
+} + +export default Menu diff --git a/playground/src/components/rtc/agent/index.module.scss b/playground/src/platform/mobile/rtc/agent/index.module.scss similarity index 100% rename from playground/src/components/rtc/agent/index.module.scss rename to playground/src/platform/mobile/rtc/agent/index.module.scss diff --git a/playground/src/components/rtc/agent/index.tsx b/playground/src/platform/mobile/rtc/agent/index.tsx similarity index 100% rename from playground/src/components/rtc/agent/index.tsx rename to playground/src/platform/mobile/rtc/agent/index.tsx diff --git a/playground/src/components/rtc/audioVisualizer/index.module.scss b/playground/src/platform/mobile/rtc/audioVisualizer/index.module.scss similarity index 100% rename from playground/src/components/rtc/audioVisualizer/index.module.scss rename to playground/src/platform/mobile/rtc/audioVisualizer/index.module.scss diff --git a/playground/src/components/rtc/audioVisualizer/index.tsx b/playground/src/platform/mobile/rtc/audioVisualizer/index.tsx similarity index 100% rename from playground/src/components/rtc/audioVisualizer/index.tsx rename to playground/src/platform/mobile/rtc/audioVisualizer/index.tsx diff --git a/playground/src/components/rtc/camSection/camSelect/index.module.scss b/playground/src/platform/mobile/rtc/camSection/camSelect/index.module.scss similarity index 100% rename from playground/src/components/rtc/camSection/camSelect/index.module.scss rename to playground/src/platform/mobile/rtc/camSection/camSelect/index.module.scss diff --git a/playground/src/components/rtc/camSection/camSelect/index.tsx b/playground/src/platform/mobile/rtc/camSection/camSelect/index.tsx similarity index 100% rename from playground/src/components/rtc/camSection/camSelect/index.tsx rename to playground/src/platform/mobile/rtc/camSection/camSelect/index.tsx diff --git a/playground/src/platform/mobile/rtc/camSection/index.module.scss b/playground/src/platform/mobile/rtc/camSection/index.module.scss new file mode 100644 index 00000000..76f4ad1e --- /dev/null +++ b/playground/src/platform/mobile/rtc/camSection/index.module.scss @@ -0,0 +1,54 @@ +.camera { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .title { + margin-bottom: 10px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 500; + line-height: 150%; + letter-spacing: 0.449px; + } + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + .select { + flex: 0 0 auto; + width: 200px; + } + } + + .view { + position: relative; + margin-top: 12px; + min-height: 210px; + height: 210px; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/playground/src/platform/mobile/rtc/camSection/index.tsx b/playground/src/platform/mobile/rtc/camSection/index.tsx new file mode 100644 index 00000000..2bd0e8db --- /dev/null +++ b/playground/src/platform/mobile/rtc/camSection/index.tsx @@ -0,0 +1,42 @@ +"use client" + +import CamSelect from "./camSelect" +import { CamIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { ICameraVideoTrack } from 'agora-rtc-sdk-ng'; +import { LocalStreamPlayer } from "../streamPlayer" +import { useState, useEffect, useMemo } from 'react'; +import { useSmallScreen } from "@/common" + +interface CamSectionProps { + videoTrack?: ICameraVideoTrack +} + +const CamSection = (props: CamSectionProps) => { + const { videoTrack } = props + const [videoMute, setVideoMute] = useState(false) + + useEffect(() => { + videoTrack?.setMuted(videoMute) + }, [videoTrack, videoMute]) + + const onClickMute = () => { + setVideoMute(!videoMute) + } + + return
+
CAMERA
+
+ + + + +
+
+ +
+
+} + + +export default CamSection; diff --git a/playground/src/components/rtc/index.module.scss b/playground/src/platform/mobile/rtc/index.module.scss similarity index 79% rename from playground/src/components/rtc/index.module.scss rename to playground/src/platform/mobile/rtc/index.module.scss index 4aa3f5c7..ff7b7958 100644 --- a/playground/src/components/rtc/index.module.scss +++ b/playground/src/platform/mobile/rtc/index.module.scss @@ -15,14 +15,20 @@ height: 40px; padding: 0px 16px; align-items: center; - gap: 238px; align-self: stretch; - color: var(--Grey-300, #EAECF0); - font-size: 14px; - font-weight: 600; - line-height: 150%; - letter-spacing: 0.449px; border-bottom: 1px solid #272A2F; + + .text { + flex: 1 1 auto; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + color: var(--Grey-300, #EAECF0); + } + + .voiceSelect { + flex: 0 0 120px; + } } .you { @@ -44,6 +50,6 @@ text-align: center; } - + } } diff --git a/playground/src/components/rtc/index.tsx b/playground/src/platform/mobile/rtc/index.tsx similarity index 76% rename from playground/src/components/rtc/index.tsx rename to playground/src/platform/mobile/rtc/index.tsx index b5f42cce..bc15c070 100644 --- a/playground/src/components/rtc/index.tsx +++ b/playground/src/platform/mobile/rtc/index.tsx @@ -1,21 +1,25 @@ "use client" import { ICameraVideoTrack, IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" -import { useAppSelector, useAppDispatch } from "@/common" +import { useAppSelector, useAppDispatch, VOICE_OPTIONS } from "@/common" import { ITextItem } from "@/types" import { rtcManager, IUserTracks, IRtcUser } from "@/manager" -import { setRoomConnected, addChatItem } from "@/store/reducers/global" +import { setRoomConnected, addChatItem, setVoiceType } from "@/store/reducers/global" import MicSection from "./micSection" import CamSection from "./camSection" import Agent from "./agent" import styles from "./index.module.scss" -import { useRef, useEffect, useState } from "react" +import { useRef, useEffect, useState, Fragment } from "react" +import { VoiceIcon } from "@/components/icons" +import CustomSelect from "@/components/customSelect" let hasInit = false const Rtc = () => { const dispatch = useAppDispatch() const options = useAppSelector(state => state.global.options) + const voiceType = useAppSelector(state => state.global.voiceType) + const agentConnected = useAppSelector(state => state.global.agentConnected) const { userId, channel } = options const [videoTrack, setVideoTrack] = useState() const [audioTrack, setAudioTrack] = useState() @@ -93,8 +97,20 @@ const Rtc = () => { } } + const onVoiceChange = (value: any) => { + dispatch(setVoiceType(value)) + } + + return
-
Audio & Video
+
+ Audio & Video + } + options={VOICE_OPTIONS} onChange={onVoiceChange}> +
{/* agent */} {/* you */} diff --git a/playground/src/platform/mobile/rtc/micSection/index.module.scss b/playground/src/platform/mobile/rtc/micSection/index.module.scss new file mode 100644 index 00000000..60cc6fe1 --- /dev/null +++ b/playground/src/platform/mobile/rtc/micSection/index.module.scss @@ -0,0 +1,58 @@ +.microphone { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .title { + margin-bottom: 10px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 500; + line-height: 150%; + letter-spacing: 0.449px; + } + + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + + } + + .view { + margin-top: 12px; + display: flex; + height: 120px; + padding: 24px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + align-self: stretch; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/playground/src/platform/mobile/rtc/micSection/index.tsx b/playground/src/platform/mobile/rtc/micSection/index.tsx new file mode 100644 index 00000000..3c739159 --- /dev/null +++ b/playground/src/platform/mobile/rtc/micSection/index.tsx @@ -0,0 +1,70 @@ +"use client" + +import { useEffect, useMemo, useState } from "react" +import { useMultibandTrackVolume, useSmallScreen } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { MicIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import MicSelect from "./micSelect"; + +interface MicSectionProps { + audioTrack?: IMicrophoneAudioTrack +} + +const MicSection = (props: MicSectionProps) => { + const { audioTrack } = props + const [audioMute, setAudioMute] = useState(false) + const [mediaStreamTrack, setMediaStreamTrack] = useState() + + + + useEffect(() => { + audioTrack?.on("track-updated", onAudioTrackupdated) + if (audioTrack) { + setMediaStreamTrack(audioTrack.getMediaStreamTrack()) + } + + return () => { + audioTrack?.off("track-updated", onAudioTrackupdated) + } + }, [audioTrack]) + + useEffect(() => { + audioTrack?.setMuted(audioMute) + }, [audioTrack, audioMute]) + + const subscribedVolumes = useMultibandTrackVolume(mediaStreamTrack, 20); + + const onAudioTrackupdated = (track: MediaStreamTrack) => { + console.log("[test] audio track updated", track) + setMediaStreamTrack(track) + } + + const onClickMute = () => { + setAudioMute(!audioMute) + } + + return
+
MICROPHONE
+
+ + + + +
+
+ +
+
+} + + +export default MicSection; diff --git a/playground/src/components/rtc/micSection/micSelect/index.module.scss b/playground/src/platform/mobile/rtc/micSection/micSelect/index.module.scss similarity index 100% rename from playground/src/components/rtc/micSection/micSelect/index.module.scss rename to playground/src/platform/mobile/rtc/micSection/micSelect/index.module.scss diff --git a/playground/src/components/rtc/micSection/micSelect/index.tsx b/playground/src/platform/mobile/rtc/micSection/micSelect/index.tsx similarity index 100% rename from playground/src/components/rtc/micSection/micSelect/index.tsx rename to playground/src/platform/mobile/rtc/micSection/micSelect/index.tsx diff --git a/playground/src/components/rtc/streamPlayer/index.module.scss b/playground/src/platform/mobile/rtc/streamPlayer/index.module.scss similarity index 100% rename from playground/src/components/rtc/streamPlayer/index.module.scss rename to playground/src/platform/mobile/rtc/streamPlayer/index.module.scss diff --git a/playground/src/components/rtc/streamPlayer/index.tsx b/playground/src/platform/mobile/rtc/streamPlayer/index.tsx similarity index 100% rename from playground/src/components/rtc/streamPlayer/index.tsx rename to playground/src/platform/mobile/rtc/streamPlayer/index.tsx diff --git a/playground/src/components/rtc/streamPlayer/localStreamPlayer.tsx b/playground/src/platform/mobile/rtc/streamPlayer/localStreamPlayer.tsx similarity index 100% rename from playground/src/components/rtc/streamPlayer/localStreamPlayer.tsx rename to playground/src/platform/mobile/rtc/streamPlayer/localStreamPlayer.tsx diff --git a/playground/src/platform/pc/chat/chatItem/index.module.scss b/playground/src/platform/pc/chat/chatItem/index.module.scss new file mode 100644 index 00000000..f28ef7ee --- /dev/null +++ b/playground/src/platform/pc/chat/chatItem/index.module.scss @@ -0,0 +1,90 @@ +.agentChatItem { + width: 100%; + display: flex; + justify-content: flex-start; + + .left { + flex: 0 0 auto; + display: flex; + width: 32px; + height: 32px; + padding: 10px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + border-radius: 200px; + background: var(--Grey-700, #475467); + + .userName { + color: var(---white, #FFF); + text-align: center; + font-size: 14px; + font-weight: 500; + line-height: 150%; + } + } + + .right { + margin-left: 12px; + flex: 1 1 auto; + + .userName { + font-size: 14px; + font-weight: 500; + line-height: 20px; + color: var(--theme-color, #667085) !important; + } + + + .agent { + color: var(--theme-color, #EAECF0) !important; + } + + } +} + +.userChatItem { + width: 100%; + display: flex; + flex-direction: column; + justify-content: flex-end; + align-items: flex-end; + + .userName { + text-align: right; + color: var(--Grey-600, #667085); + font-weight: 500; + line-height: 20px; + } + + + +} + + +.chatItem { + .text { + max-width: 80%; + width: fit-content; + margin-top: 6px; + color: #FFF; + display: flex; + padding: 8px 14px; + flex-direction: column; + justify-content: center; + align-items: flex-start; + font-size: 14px; + font-weight: 400; + line-height: 21px; + white-space: pre-wrap; + border-radius: 0px 8px 8px 8px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} + +.chatItem+.chatItem { + margin-top: 14px; +} diff --git a/playground/src/platform/pc/chat/chatItem/index.tsx b/playground/src/platform/pc/chat/chatItem/index.tsx new file mode 100644 index 00000000..6364aaea --- /dev/null +++ b/playground/src/platform/pc/chat/chatItem/index.tsx @@ -0,0 +1,51 @@ +import { IChatItem } from "@/types" +import styles from "./index.module.scss" +import { usePrevious } from "@/common" +import { use, useEffect, useMemo, useState } from "react" + +interface ChatItemProps { + data: IChatItem +} + + +const AgentChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + return
+ + Ag + + +
Agent
+
+ {text} +
+ +
+} + +const UserChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + return
+
You
+
{text}
+
+} + + +const ChatItem = (props: ChatItemProps) => { + const { data } = props + + + return ( + data.type === "agent" ? : + ); + + +} + + +export default ChatItem diff --git a/playground/src/platform/pc/chat/index.module.scss b/playground/src/platform/pc/chat/index.module.scss new file mode 100644 index 00000000..b5c2707c --- /dev/null +++ b/playground/src/platform/pc/chat/index.module.scss @@ -0,0 +1,73 @@ +.chat { + flex: 1 1 auto; + min-width: 500px; + display: flex; + flex-direction: column; + align-items: flex-start; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + overflow: hidden; + + .header { + display: flex; + height: 42px; + padding: 0px 16px; + align-items: center; + align-self: stretch; + border-bottom: 1px solid #272A2F; + + .left { + flex: 1 1 auto; + + .text { + margin-left: 4px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + height: 40px; + line-height: 40px; + letter-spacing: 0.449px; + } + + .languageSelect { + margin-left: 12px; + width: 100px; + } + } + + + .right { + flex: 0 0 230px; + } + + } + + .content { + display: flex; + padding: 12px 24px; + flex-direction: column; + align-items: flex-start; + gap: 10px; + flex: 1 0 500px; + align-self: stretch; + overflow-y: auto; + + + &::-webkit-scrollbar { + width: 6px + } + + &::-webkit-scrollbar-track { + background-color: transparent; + } + + &::-webkit-scrollbar-thumb { + background-color: #6B6B6B; + border-radius: 4px; + } + } + + +} diff --git a/playground/src/platform/pc/chat/index.tsx b/playground/src/platform/pc/chat/index.tsx new file mode 100644 index 00000000..98cdd083 --- /dev/null +++ b/playground/src/platform/pc/chat/index.tsx @@ -0,0 +1,59 @@ +"use client" + +import { ReactElement, useEffect, useRef, useState } from "react" +import ChatItem from "./chatItem" +import { + genRandomChatList, useAppDispatch, useAutoScroll, + LANGUAGE_OPTIONS, useAppSelector, +} from "@/common" +import { setLanguage } from "@/store/reducers/global" +import { Select, } from 'antd'; +import PdfSelect from "@/components/pdfSelect" + +import styles from "./index.module.scss" + + + + +const Chat = () => { + const dispatch = useAppDispatch() + const chatItems = useAppSelector(state => state.global.chatItems) + const language = useAppSelector(state => state.global.language) + const agentConnected = useAppSelector(state => state.global.agentConnected) + + // const chatItems = genRandomChatList(10) + const chatRef = useRef(null) + + + useAutoScroll(chatRef) + + + const onLanguageChange = (val: any) => { + dispatch(setLanguage(val)) + } + + + + + return
+
+ + Chat + + + + + +
+
+ {chatItems.map((item, index) => { + return + })} +
+
+} + + +export default Chat diff --git a/playground/src/platform/pc/description/index.module.scss b/playground/src/platform/pc/description/index.module.scss new file mode 100644 index 00000000..7efe9d6d --- /dev/null +++ b/playground/src/platform/pc/description/index.module.scss @@ -0,0 +1,72 @@ +.description { + position: relative; + display: flex; + padding: 12px 16px; + align-items: center; + gap: 12px; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-style: normal; + font-weight: 600; + /* 21px */ + letter-spacing: 0.449px; + } + + .text { + margin-left: 12px; + flex: 1 1 auto; + color: var(--Grey-600, #667085); + font-size: 14px; + font-style: normal; + font-weight: 400; + } + + + .btnConnect { + width: 150px; + display: flex; + padding: 8px 14px; + justify-content: center; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 6px; + background: var(--theme-color, #0888FF); + box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05); + cursor: pointer; + user-select: none; + caret-color: transparent; + box-sizing: border-box; + + .btnText { + width: 100px; + text-align: center; + color: var(---White, #FFF); + font-size: 14px; + font-weight: 500; + line-height: 20px; + } + + .btnText.disconnect { + color: var(--Error-400-T, #E95C7B); + } + + + .loading { + margin-left: 4px; + } + } + + + .btnConnect.disconnect { + background: #181A1D; + border: 1px solid var(--Error-400-T, #E95C7B); + } + +} diff --git a/playground/src/platform/pc/description/index.tsx b/playground/src/platform/pc/description/index.tsx new file mode 100644 index 00000000..f7965392 --- /dev/null +++ b/playground/src/platform/pc/description/index.tsx @@ -0,0 +1,99 @@ +import { setAgentConnected } from "@/store/reducers/global" +import { + DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, + apiStartService, apiStopService, REQUEST_URL +} from "@/common" +import { Select, Button, message, Upload } from "antd" +import { useEffect, useState, MouseEventHandler } from "react" +import { LoadingOutlined, UploadOutlined } from "@ant-design/icons" +import styles from "./index.module.scss" + +let intervalId: any + +const Description = () => { + const dispatch = useAppDispatch() + const agentConnected = useAppSelector(state => state.global.agentConnected) + const channel = useAppSelector(state => state.global.options.channel) + const userId = useAppSelector(state => state.global.options.userId) + const language = useAppSelector(state => state.global.language) + const voiceType = useAppSelector(state => state.global.voiceType) + const [loading, setLoading] = useState(false) + + useEffect(() => { + if (channel) { + checkAgentConnected() + } + }, [channel]) + + + const checkAgentConnected = async () => { + const res: any = await apiPing(channel) + if (res?.code == 0) { + dispatch(setAgentConnected(true)) + } + } + + const onClickConnect = async () => { + if (loading) { + return + } + setLoading(true) + if (agentConnected) { + await apiStopService(channel) + dispatch(setAgentConnected(false)) + message.success("Agent disconnected") + stopPing() + } else { + const res = await apiStartService({ + channel, + userId, + language, + voiceType + }) + const { code, msg } = res || {} + if (code != 0) { + if (code == "10001") { + message.error("The number of users experiencing the program simultaneously has exceeded the limit. Please try again later.") + } else { + message.error(`code:${code},msg:${msg}`) + } + setLoading(false) + throw new Error(msg) + } + dispatch(setAgentConnected(true)) + message.success("Agent connected") + startPing() + } + setLoading(false) + } + + const startPing = () => { + if (intervalId) { + stopPing() + } + intervalId = setInterval(() => { + apiPing(channel) + }, 3000) + } + + const stopPing = () => { + if (intervalId) { + clearInterval(intervalId) + intervalId = null + } + } + + return
+ Description + RTT is an AI voice assistant powered by Shengwang Agents, Deepgram, Eleven Labs, and ChatGPT. It is running on Shengwang Playground. + + + {!agentConnected ? "Connect" : "Disconnect"} + {loading ? : null} + + +
+} + + +export default Description diff --git a/playground/src/platform/pc/entry/index.module.scss b/playground/src/platform/pc/entry/index.module.scss new file mode 100644 index 00000000..f138183f --- /dev/null +++ b/playground/src/platform/pc/entry/index.module.scss @@ -0,0 +1,17 @@ +.entry { + position: relative; + height: 100%; + box-sizing: border-box; + + .content { + position: relative; + padding: 16px; + box-sizing: border-box; + + .body { + margin-top: 16px; + display: flex; + gap: 24px; + } + } +} diff --git a/playground/src/platform/pc/entry/index.tsx b/playground/src/platform/pc/entry/index.tsx new file mode 100644 index 00000000..e7acd7f1 --- /dev/null +++ b/playground/src/platform/pc/entry/index.tsx @@ -0,0 +1,22 @@ +import Chat from "../chat" +import Description from "../description" +import Rtc from "../rtc" +import Header from "../header" + +import styles from "./index.module.scss" + +const PCEntry = () => { + return
+
+
+ +
+ + +
+
+
+} + + +export default PCEntry diff --git a/playground/src/components/header/index.module.scss b/playground/src/platform/pc/header/index.module.scss similarity index 87% rename from playground/src/components/header/index.module.scss rename to playground/src/platform/pc/header/index.module.scss index d70e83f3..049d0181 100644 --- a/playground/src/components/header/index.module.scss +++ b/playground/src/platform/pc/header/index.module.scss @@ -1,8 +1,4 @@ .header { - position: fixed; - left: 0; - right: 0; - top: 0; display: flex; width: 100%; height: 48px; @@ -30,6 +26,9 @@ } .content { + display: flex; + align-items: center; + justify-content: center; height: 48px; flex: 1 1 auto; color: var(--Grey-300, #EAECF0); @@ -38,10 +37,13 @@ line-height: 48px; letter-spacing: 0.449px; text-align: center; + + .text { + margin-left: 4px; + } } .githubWrapper { - cursor: pointer; margin-right: 12px; } } diff --git a/playground/src/platform/pc/header/index.tsx b/playground/src/platform/pc/header/index.tsx new file mode 100644 index 00000000..15d3fcef --- /dev/null +++ b/playground/src/platform/pc/header/index.tsx @@ -0,0 +1,46 @@ +"use client" + +import { useAppSelector, GITHUB_URL, useSmallScreen } from "@/common" +import Network from "./network" +import InfoPopover from "./infoPopover" +import StylePopover from "./stylePopover" +import { GithubIcon, LogoIcon, InfoIcon, ColorPickerIcon } from "@/components/icons" + +import styles from "./index.module.scss" + +const Header = () => { + const themeColor = useAppSelector(state => state.global.themeColor) + const options = useAppSelector(state => state.global.options) + const { channel } = options + + + const onClickGithub = () => { + if (typeof window !== "undefined") { + window.open(GITHUB_URL, "_blank") + } + } + + + + return
+ + + + + + + Channel Name: {channel} + + + + + + + + + +
+} + + +export default Header diff --git a/playground/src/components/setting/status/index.module.scss b/playground/src/platform/pc/header/infoPopover/index.module.scss similarity index 83% rename from playground/src/components/setting/status/index.module.scss rename to playground/src/platform/pc/header/infoPopover/index.module.scss index aa84abeb..cd3f72f8 100644 --- a/playground/src/components/setting/status/index.module.scss +++ b/playground/src/platform/pc/header/infoPopover/index.module.scss @@ -1,11 +1,10 @@ -.status { +.info { display: flex; - padding: 24px 16px; + padding: 12px 16px; flex-direction: column; align-items: flex-start; - gap: 12px; + gap: 8px; align-self: stretch; - border-bottom: 1px solid #272A2F; .title { color: var(--Grey-300, #EAECF0); @@ -35,4 +34,10 @@ line-height: 150%; } } + + .slider { + height: 1px; + width: 100%; + background-color: #0D0F12; + } } diff --git a/playground/src/platform/pc/header/infoPopover/index.tsx b/playground/src/platform/pc/header/infoPopover/index.tsx new file mode 100644 index 00000000..cd451418 --- /dev/null +++ b/playground/src/platform/pc/header/infoPopover/index.tsx @@ -0,0 +1,57 @@ +import { useMemo } from "react" +import { useAppSelector } from "@/common" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface InfoPopoverProps { + children?: React.ReactNode +} + +const InfoPopover = (props: InfoPopoverProps) => { + const { children } = props + const options = useAppSelector(state => state.global.options) + const { channel, userId } = options + + const roomConnected = useAppSelector(state => state.global.roomConnected) + const agentConnected = useAppSelector(state => state.global.agentConnected) + + const roomConnectedText = useMemo(() => { + return roomConnected ? "TRUE" : "FALSE" + }, [roomConnected]) + + const agentConnectedText = useMemo(() => { + return agentConnected ? "TRUE" : "FALSE" + }, [agentConnected]) + + + + const content =
+
INFO
+
+ Room + {channel} +
+
+ Participant + {userId} +
+
+
STATUS
+
+
Room connected
+
{roomConnectedText}
+
+
+
Agent connected
+
{agentConnectedText}
+
+
+ + + return {children} + +} + +export default InfoPopover diff --git a/playground/src/platform/pc/header/network/index.module.scss b/playground/src/platform/pc/header/network/index.module.scss new file mode 100644 index 00000000..e69de29b diff --git a/playground/src/platform/pc/header/network/index.tsx b/playground/src/platform/pc/header/network/index.tsx new file mode 100644 index 00000000..92b4e33b --- /dev/null +++ b/playground/src/platform/pc/header/network/index.tsx @@ -0,0 +1,37 @@ +"use client"; + +import React from "react"; +import { rtcManager } from "@/manager" +import { NetworkQuality } from "agora-rtc-sdk-ng" +import { useEffect, useState } from "react" +import { NetworkIcon } from "@/components/icons" + +interface NetworkProps { + style?: React.CSSProperties +} + +const NetWork = (props: NetworkProps) => { + const { style } = props + + const [networkQuality, setNetworkQuality] = useState() + + useEffect(() => { + rtcManager.on("networkQuality", onNetworkQuality) + + return () => { + rtcManager.off("networkQuality", onNetworkQuality) + } + }, []) + + const onNetworkQuality = (quality: NetworkQuality) => { + setNetworkQuality(quality) + } + + return ( + + + + ) +} + +export default NetWork diff --git a/playground/src/platform/pc/header/stylePopover/colorPicker/index.module.scss b/playground/src/platform/pc/header/stylePopover/colorPicker/index.module.scss new file mode 100644 index 00000000..405e7781 --- /dev/null +++ b/playground/src/platform/pc/header/stylePopover/colorPicker/index.module.scss @@ -0,0 +1,24 @@ +.colorPicker { + height: 24px; + display: flex; + align-items: center; + + :global(.react-colorful) { + width: 220px; + height: 8px; + } + + :global(.react-colorful__saturation) { + display: none; + } + + :global(.react-colorful__hue) { + border-radius: 8px !important; + height: 8px; + } + + :global(.react-colorful__pointer) { + width: 24px; + height: 24px; + } +} diff --git a/playground/src/platform/pc/header/stylePopover/colorPicker/index.tsx b/playground/src/platform/pc/header/stylePopover/colorPicker/index.tsx new file mode 100644 index 00000000..28163d77 --- /dev/null +++ b/playground/src/platform/pc/header/stylePopover/colorPicker/index.tsx @@ -0,0 +1,22 @@ +"use client" + +import { HexColorPicker } from "react-colorful"; +import { useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import styles from "./index.module.scss"; + +const ColorPicker = () => { + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + const onColorChange = (color: string) => { + console.log(color); + dispatch(setThemeColor(color)) + }; + + return
+ +
+}; + +export default ColorPicker; diff --git a/playground/src/platform/pc/header/stylePopover/index.module.scss b/playground/src/platform/pc/header/stylePopover/index.module.scss new file mode 100644 index 00000000..98c7f182 --- /dev/null +++ b/playground/src/platform/pc/header/stylePopover/index.module.scss @@ -0,0 +1,51 @@ +.info { + display: flex; + padding: 12px 16px; + flex-direction: column; + align-items: flex-start; + gap: 16px; + align-self: stretch; + + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .color { + font-size: 0; + white-space: nowrap; + + .item { + position: relative; + display: inline-block; + width: 28px; + height: 28px; + border-radius: 4px; + border: 2px solid transparent; + font-size: 0; + cursor: pointer; + + .inner { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + width: 18px; + height: 18px; + border-radius: 2px; + box-sizing: border-box; + } + } + + .item+.item { + margin-left: 12px; + } + + } + + +} diff --git a/playground/src/platform/pc/header/stylePopover/index.tsx b/playground/src/platform/pc/header/stylePopover/index.tsx new file mode 100644 index 00000000..f8508323 --- /dev/null +++ b/playground/src/platform/pc/header/stylePopover/index.tsx @@ -0,0 +1,54 @@ +import { useMemo } from "react" +import { COLOR_LIST, useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import ColorPicker from "./colorPicker" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface StylePopoverProps { + children?: React.ReactNode +} + +const StylePopover = (props: StylePopoverProps) => { + const { children } = props + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + + const onClickColor = (index: number) => { + const target = COLOR_LIST[index] + if (target.active !== themeColor) { + dispatch(setThemeColor(target.active)) + } + } + + const content =
+
STYLE
+
+ { + COLOR_LIST.map((item, index) => { + return onClickColor(index)} + className={styles.item} + key={index}> + + + }) + } +
+ +
+ + + return {children} + +} + +export default StylePopover diff --git a/playground/src/platform/pc/rtc/agent/index.module.scss b/playground/src/platform/pc/rtc/agent/index.module.scss new file mode 100644 index 00000000..fa3ae2ec --- /dev/null +++ b/playground/src/platform/pc/rtc/agent/index.module.scss @@ -0,0 +1,31 @@ +.agent { + position: relative; + display: flex; + height: 292px; + padding: 20px 16px; + flex-direction: column; + justify-content: flex-start; + align-items: center; + align-self: stretch; + background: linear-gradient(154deg, rgba(27, 66, 166, 0.16) 0%, rgba(27, 45, 140, 0.00) 18%), linear-gradient(153deg, rgba(23, 24, 28, 0.00) 53.75%, #11174E 100%), #0F0F11; + box-shadow: 0px 3.999px 48.988px 0px rgba(0, 7, 72, 0.12); + backdrop-filter: blur(7); + box-sizing: border-box; + + .text { + margin-top: 50px; + color: var(--theme-color, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .view { + margin-top: 32px; + display: flex; + align-items: center; + justify-content: center; + height: 56px; + } +} diff --git a/playground/src/platform/pc/rtc/agent/index.tsx b/playground/src/platform/pc/rtc/agent/index.tsx new file mode 100644 index 00000000..a7fd7944 --- /dev/null +++ b/playground/src/platform/pc/rtc/agent/index.tsx @@ -0,0 +1,34 @@ +"use client" + +import { useAppSelector, useMultibandTrackVolume } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import styles from "./index.module.scss" + +interface AgentProps { + audioTrack?: IMicrophoneAudioTrack +} + +const Agent = (props: AgentProps) => { + const { audioTrack } = props + + const subscribedVolumes = useMultibandTrackVolume(audioTrack, 12); + + return
+
Agent
+
+ +
+
+ +} + + +export default Agent; diff --git a/playground/src/platform/pc/rtc/audioVisualizer/index.module.scss b/playground/src/platform/pc/rtc/audioVisualizer/index.module.scss new file mode 100644 index 00000000..1beae944 --- /dev/null +++ b/playground/src/platform/pc/rtc/audioVisualizer/index.module.scss @@ -0,0 +1,17 @@ +.audioVisualizer { + display: flex; + justify-content: center; + align-items: center; + + + .item {} + + .agent { + background-color: var(--theme-color, #EAECF0); + box-shadow: 0 0 10px var(--theme-color, #EAECF0); + } + + .user { + background-color: var(--Grey-300, #EAECF0); + } +} diff --git a/playground/src/platform/pc/rtc/audioVisualizer/index.tsx b/playground/src/platform/pc/rtc/audioVisualizer/index.tsx new file mode 100644 index 00000000..bc21f554 --- /dev/null +++ b/playground/src/platform/pc/rtc/audioVisualizer/index.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useState, useEffect } from "react" +import styles from "./index.module.scss" + +interface AudioVisualizerProps { + type: "agent" | "user"; + frequencies: Float32Array[]; + gap: number; + barWidth: number; + minBarHeight: number; + maxBarHeight: number + borderRadius: number; +} + + +const AudioVisualizer = (props: AudioVisualizerProps) => { + const { frequencies, gap, barWidth, minBarHeight, maxBarHeight, borderRadius, type } = props; + + const summedFrequencies = frequencies.map((bandFrequencies) => { + const sum = bandFrequencies.reduce((a, b) => a + b, 0) + if (sum <= 0) { + return 0 + } + return Math.sqrt(sum / bandFrequencies.length); + }); + + return
{ + summedFrequencies.map((frequency, index) => { + + const style = { + height: minBarHeight + frequency * (maxBarHeight - minBarHeight) + "px", + borderRadius: borderRadius + "px", + width: barWidth + "px", + transition: + "background-color 0.35s ease-out, transform 0.25s ease-out", + // transform: transform, + } + + return + }) + }
+} + + +export default AudioVisualizer; diff --git a/playground/src/platform/pc/rtc/camSection/camSelect/index.module.scss b/playground/src/platform/pc/rtc/camSection/camSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/playground/src/platform/pc/rtc/camSection/camSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/playground/src/platform/pc/rtc/camSection/camSelect/index.tsx b/playground/src/platform/pc/rtc/camSection/camSelect/index.tsx new file mode 100644 index 00000000..33a5e003 --- /dev/null +++ b/playground/src/platform/pc/rtc/camSection/camSelect/index.tsx @@ -0,0 +1,57 @@ +"use client" + +import AgoraRTC, { ICameraVideoTrack } from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" + +import styles from "./index.module.scss" + +interface CamSelectProps { + videoTrack?: ICameraVideoTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const CamSelect = (props: CamSelectProps) => { + const { videoTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (videoTrack) { + const label = videoTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getCameras().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [videoTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (videoTrack) { + await videoTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default CamSelect diff --git a/playground/src/components/rtc/camSection/index.module.scss b/playground/src/platform/pc/rtc/camSection/index.module.scss similarity index 100% rename from playground/src/components/rtc/camSection/index.module.scss rename to playground/src/platform/pc/rtc/camSection/index.module.scss diff --git a/playground/src/components/rtc/camSection/index.tsx b/playground/src/platform/pc/rtc/camSection/index.tsx similarity index 94% rename from playground/src/components/rtc/camSection/index.tsx rename to playground/src/platform/pc/rtc/camSection/index.tsx index f5b62e06..99e5392c 100644 --- a/playground/src/components/rtc/camSection/index.tsx +++ b/playground/src/platform/pc/rtc/camSection/index.tsx @@ -15,11 +15,11 @@ interface CamSectionProps { const CamSection = (props: CamSectionProps) => { const { videoTrack } = props const [videoMute, setVideoMute] = useState(false) - const {xs} = useSmallScreen() + const { xs } = useSmallScreen() - const CamText = useMemo(()=>{ + const CamText = useMemo(() => { return xs ? "CAM" : "CAMERA" - },[xs]) + }, [xs]) useEffect(() => { videoTrack?.setMuted(videoMute) diff --git a/playground/src/platform/pc/rtc/index.module.scss b/playground/src/platform/pc/rtc/index.module.scss new file mode 100644 index 00000000..b62025c5 --- /dev/null +++ b/playground/src/platform/pc/rtc/index.module.scss @@ -0,0 +1,55 @@ +.rtc { + flex: 0 0 420px; + display: flex; + flex-direction: column; + align-items: flex-start; + flex-shrink: 0; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + box-sizing: border-box; + + .header { + display: flex; + height: 42px; + padding: 0px 16px; + align-items: center; + align-self: stretch; + border-bottom: 1px solid #272A2F; + + .text { + flex: 1 1 auto; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + color: var(--Grey-300, #EAECF0); + } + + .voiceSelect { + flex: 0 0 120px; + } + } + + .you { + display: flex; + padding: 24px 16px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 24px; + align-self: stretch; + border-top: 1px solid #272A2F; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + text-align: center; + } + + + } +} diff --git a/playground/src/platform/pc/rtc/index.tsx b/playground/src/platform/pc/rtc/index.tsx new file mode 100644 index 00000000..1195ca0f --- /dev/null +++ b/playground/src/platform/pc/rtc/index.tsx @@ -0,0 +1,128 @@ +"use client" + +import { ICameraVideoTrack, IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" +import { useAppSelector, useAppDispatch, VOICE_OPTIONS } from "@/common" +import { ITextItem } from "@/types" +import { rtcManager, IUserTracks, IRtcUser } from "@/manager" +import { setRoomConnected, addChatItem, setVoiceType } from "@/store/reducers/global" +import MicSection from "./micSection" +import CamSection from "./camSection" +import Agent from "./agent" +import styles from "./index.module.scss" +import { useRef, useEffect, useState, Fragment } from "react" +import { VoiceIcon } from "@/components/icons" +import CustomSelect from "@/components/customSelect" + +let hasInit = false + +const Rtc = () => { + const dispatch = useAppDispatch() + const options = useAppSelector(state => state.global.options) + const voiceType = useAppSelector(state => state.global.voiceType) + const agentConnected = useAppSelector(state => state.global.agentConnected) + const { userId, channel } = options + const [videoTrack, setVideoTrack] = useState() + const [audioTrack, setAudioTrack] = useState() + const [remoteuser, setRemoteUser] = useState() + + useEffect(() => { + if (!options.channel) { + return + } + if (hasInit) { + return + } + + init() + + return () => { + if (hasInit) { + destory() + } + } + }, [options.channel]) + + + const init = async () => { + console.log("[test] init") + rtcManager.on("localTracksChanged", onLocalTracksChanged) + rtcManager.on("textChanged", onTextChanged) + rtcManager.on("remoteUserChanged", onRemoteUserChanged) + await rtcManager.createTracks() + await rtcManager.join({ + channel, + userId + }) + await rtcManager.publish() + dispatch(setRoomConnected(true)) + hasInit = true + } + + const destory = async () => { + console.log("[test] destory") + rtcManager.off("textChanged", onTextChanged) + rtcManager.off("localTracksChanged", onLocalTracksChanged) + rtcManager.off("remoteUserChanged", onRemoteUserChanged) + await rtcManager.destroy() + dispatch(setRoomConnected(false)) + hasInit = false + } + + const onRemoteUserChanged = (user: IRtcUser) => { + console.log("[test] onRemoteUserChanged", user) + setRemoteUser(user) + } + + const onLocalTracksChanged = (tracks: IUserTracks) => { + console.log("[test] onLocalTracksChanged", tracks) + const { videoTrack, audioTrack } = tracks + if (videoTrack) { + setVideoTrack(videoTrack) + } + if (audioTrack) { + setAudioTrack(audioTrack) + } + } + + const onTextChanged = (text: ITextItem) => { + if (text.dataType == "transcribe") { + const isAgent = Number(text.uid) != Number(userId) + dispatch(addChatItem({ + userId: text.uid, + text: text.text, + type: isAgent ? "agent" : "user", + isFinal: text.isFinal, + time: text.time + })) + } + } + + const onVoiceChange = (value: any) => { + dispatch(setVoiceType(value)) + } + + + return
+
+ Audio & Video + } + options={VOICE_OPTIONS} onChange={onVoiceChange}> +
+ {/* agent */} + + {/* you */} +
+
You
+ {/* microphone */} + + {/* camera */} + +
+
+} + + +export default Rtc; diff --git a/playground/src/components/rtc/micSection/index.module.scss b/playground/src/platform/pc/rtc/micSection/index.module.scss similarity index 100% rename from playground/src/components/rtc/micSection/index.module.scss rename to playground/src/platform/pc/rtc/micSection/index.module.scss diff --git a/playground/src/components/rtc/micSection/index.tsx b/playground/src/platform/pc/rtc/micSection/index.tsx similarity index 98% rename from playground/src/components/rtc/micSection/index.tsx rename to playground/src/platform/pc/rtc/micSection/index.tsx index e1e8a10b..6d97f3e2 100644 --- a/playground/src/components/rtc/micSection/index.tsx +++ b/playground/src/platform/pc/rtc/micSection/index.tsx @@ -18,7 +18,7 @@ const MicSection = (props: MicSectionProps) => { const [mediaStreamTrack, setMediaStreamTrack] = useState() const { xs } = useSmallScreen() - const MicText = useMemo(() => { + const MicText = useMemo(() => { return xs ? "MIC" : "MICROPHONE" }, [xs]) diff --git a/playground/src/platform/pc/rtc/micSection/micSelect/index.module.scss b/playground/src/platform/pc/rtc/micSection/micSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/playground/src/platform/pc/rtc/micSection/micSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/playground/src/platform/pc/rtc/micSection/micSelect/index.tsx b/playground/src/platform/pc/rtc/micSection/micSelect/index.tsx new file mode 100644 index 00000000..efc842b5 --- /dev/null +++ b/playground/src/platform/pc/rtc/micSection/micSelect/index.tsx @@ -0,0 +1,58 @@ +"use client" + +import AgoraRTC from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" +import { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" + +import styles from "./index.module.scss" + +interface MicSelectProps { + audioTrack?: IMicrophoneAudioTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const MicSelect = (props: MicSelectProps) => { + const { audioTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (audioTrack) { + const label = audioTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getMicrophones().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [audioTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (audioTrack) { + await audioTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default MicSelect diff --git a/playground/src/platform/pc/rtc/streamPlayer/index.module.scss b/playground/src/platform/pc/rtc/streamPlayer/index.module.scss new file mode 100644 index 00000000..b1c57c10 --- /dev/null +++ b/playground/src/platform/pc/rtc/streamPlayer/index.module.scss @@ -0,0 +1,6 @@ +.streamPlayer { + position: relative; + width: 100%; + height: 100%; + overflow: hidden; +} diff --git a/playground/src/platform/pc/rtc/streamPlayer/index.tsx b/playground/src/platform/pc/rtc/streamPlayer/index.tsx new file mode 100644 index 00000000..ba78e377 --- /dev/null +++ b/playground/src/platform/pc/rtc/streamPlayer/index.tsx @@ -0,0 +1 @@ +export * from "./localStreamPlayer" diff --git a/playground/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx b/playground/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx new file mode 100644 index 00000000..e3e7f06a --- /dev/null +++ b/playground/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx @@ -0,0 +1,46 @@ +"use client" + +import { + ICameraVideoTrack, + IMicrophoneAudioTrack, + IRemoteAudioTrack, + IRemoteVideoTrack, + VideoPlayerConfig, +} from "agora-rtc-sdk-ng" +import { useRef, useState, useLayoutEffect, forwardRef, useEffect, useMemo } from "react" + +import styles from "./index.module.scss" + +interface StreamPlayerProps { + videoTrack?: ICameraVideoTrack + audioTrack?: IMicrophoneAudioTrack + style?: React.CSSProperties + fit?: "cover" | "contain" | "fill" + onClick?: () => void + mute?: boolean +} + +export const LocalStreamPlayer = forwardRef((props: StreamPlayerProps, ref) => { + const { videoTrack, audioTrack, mute = false, style = {}, fit = "cover", onClick = () => { } } = props + const vidDiv = useRef(null) + + useLayoutEffect(() => { + const config = { fit } as VideoPlayerConfig + if (mute) { + videoTrack?.stop() + } else { + if (!videoTrack?.isPlaying) { + videoTrack?.play(vidDiv.current!, config) + } + } + + return () => { + videoTrack?.stop() + } + }, [videoTrack, fit, mute]) + + // local audio track need not to be played + // useLayoutEffect(() => {}, [audioTrack, localAudioMute]) + + return
+}) diff --git a/playground/src/protobuf/SttMessage001.js b/playground/src/protobuf/SttMessage001.js deleted file mode 100644 index 16c2ebe7..00000000 --- a/playground/src/protobuf/SttMessage001.js +++ /dev/null @@ -1,143 +0,0 @@ -/* eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars */ -;(function (global, factory) { - /* global define, require, module */ - - /* AMD */ if (typeof define === "function" && define.amd) define(["protobufjs/light"], factory) - /* CommonJS */ else if ( - typeof require === "function" && - typeof module === "object" && - module && - module.exports - ) - module.exports = factory(require("protobufjs/light")) -})(this, function ($protobuf) { - "use strict" - - var $root = ($protobuf.roots.default || ($protobuf.roots.default = new $protobuf.Root())).addJSON( - { - Agora: { - nested: { - SpeechToText: { - options: { - objc_class_prefix: "Stt", - csharp_namespace: "AgoraSTTSample.Protobuf", - java_package: "io.agora.rtc.speech2text", - java_outer_classname: "AgoraSpeech2TextProtobuffer", - }, - nested: { - Text: { - fields: { - vendor: { - type: "int32", - id: 1, - }, - version: { - type: "int32", - id: 2, - }, - seqnum: { - type: "int32", - id: 3, - }, - uid: { - type: "int64", - id: 4, - }, - flag: { - type: "int32", - id: 5, - }, - time: { - type: "int64", - id: 6, - }, - lang: { - type: "int32", - id: 7, - }, - starttime: { - type: "int32", - id: 8, - }, - offtime: { - type: "int32", - id: 9, - }, - words: { - rule: "repeated", - type: "Word", - id: 10, - }, - endOfSegment: { - type: "bool", - id: 11, - }, - durationMs: { - type: "int32", - id: 12, - }, - dataType: { - type: "string", - id: 13, - }, - trans: { - rule: "repeated", - type: "Translation", - id: 14, - }, - culture: { - type: "string", - id: 15, - }, - }, - }, - Word: { - fields: { - text: { - type: "string", - id: 1, - }, - startMs: { - type: "int32", - id: 2, - }, - durationMs: { - type: "int32", - id: 3, - }, - isFinal: { - type: "bool", - id: 4, - }, - confidence: { - type: "double", - id: 5, - }, - }, - }, - Translation: { - fields: { - isFinal: { - type: "bool", - id: 1, - }, - lang: { - type: "string", - id: 2, - }, - texts: { - rule: "repeated", - type: "string", - id: 3, - }, - }, - }, - }, - }, - }, - }, - }, - ) - - return $root -}) diff --git a/playground/src/protobuf/SttMessage002.js b/playground/src/protobuf/SttMessage002.js deleted file mode 100644 index 4634fccc..00000000 --- a/playground/src/protobuf/SttMessage002.js +++ /dev/null @@ -1,130 +0,0 @@ -/* eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars */ -import * as $protobuf from "protobufjs/light" - -const $root = ($protobuf.roots.default || ($protobuf.roots.default = new $protobuf.Root())).addJSON( - { - Agora: { - nested: { - SpeechToText: { - options: { - objc_class_prefix: "Stt", - csharp_namespace: "AgoraSTTSample.Protobuf", - java_package: "io.agora.rtc.speech2text", - java_outer_classname: "AgoraSpeech2TextProtobuffer", - }, - nested: { - Text: { - fields: { - vendor: { - type: "int32", - id: 1, - }, - version: { - type: "int32", - id: 2, - }, - seqnum: { - type: "int32", - id: 3, - }, - uid: { - type: "int64", - id: 4, - }, - flag: { - type: "int32", - id: 5, - }, - time: { - type: "int64", - id: 6, - }, - lang: { - type: "int32", - id: 7, - }, - starttime: { - type: "int32", - id: 8, - }, - offtime: { - type: "int32", - id: 9, - }, - words: { - rule: "repeated", - type: "Word", - id: 10, - }, - endOfSegment: { - type: "bool", - id: 11, - }, - durationMs: { - type: "int32", - id: 12, - }, - dataType: { - type: "string", - id: 13, - }, - trans: { - rule: "repeated", - type: "Translation", - id: 14, - }, - culture: { - type: "string", - id: 15, - }, - }, - }, - Word: { - fields: { - text: { - type: "string", - id: 1, - }, - startMs: { - type: "int32", - id: 2, - }, - durationMs: { - type: "int32", - id: 3, - }, - isFinal: { - type: "bool", - id: 4, - }, - confidence: { - type: "double", - id: 5, - }, - }, - }, - Translation: { - fields: { - isFinal: { - type: "bool", - id: 1, - }, - lang: { - type: "string", - id: 2, - }, - texts: { - rule: "repeated", - type: "string", - id: 3, - }, - }, - }, - }, - }, - }, - }, - }, -) - -export { $root as default } diff --git a/playground/src/protobuf/SttMessage003.js b/playground/src/protobuf/SttMessage003.js deleted file mode 100644 index 2afd970c..00000000 --- a/playground/src/protobuf/SttMessage003.js +++ /dev/null @@ -1,1161 +0,0 @@ -/* eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars */ -import * as $protobuf from "protobufjs/minimal" - -// Common aliases -const $Reader = $protobuf.Reader -const $Writer = $protobuf.Writer -const $util = $protobuf.util - -// Exported root namespace -const $root = $protobuf.roots.default || ($protobuf.roots.default = {}) - -export const Agora = ($root.Agora = (() => { - /** - * Namespace Agora. - * @exports Agora - * @namespace - */ - const Agora = {} - - Agora.SpeechToText = (function () { - /** - * Namespace SpeechToText. - * @memberof Agora - * @namespace - */ - const SpeechToText = {} - - SpeechToText.Text = (function () { - /** - * Properties of a Text. - * @memberof Agora.SpeechToText - * @interface IText - * @property {number|null} [vendor] Text vendor - * @property {number|null} [version] Text version - * @property {number|null} [seqnum] Text seqnum - * @property {number|Long|null} [uid] Text uid - * @property {number|null} [flag] Text flag - * @property {number|Long|null} [time] Text time - * @property {number|null} [lang] Text lang - * @property {number|null} [starttime] Text starttime - * @property {number|null} [offtime] Text offtime - * @property {Array.|null} [words] Text words - * @property {boolean|null} [endOfSegment] Text endOfSegment - * @property {number|null} [durationMs] Text durationMs - * @property {string|null} [dataType] Text dataType - * @property {Array.|null} [trans] Text trans - * @property {string|null} [culture] Text culture - */ - - /** - * Constructs a new Text. - * @memberof Agora.SpeechToText - * @classdesc Represents a Text. - * @implements IText - * @constructor - * @param {Agora.SpeechToText.IText=} [properties] Properties to set - */ - function Text(properties) { - this.words = [] - this.trans = [] - if (properties) - for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i) - if (properties[keys[i]] != null) this[keys[i]] = properties[keys[i]] - } - - /** - * Text vendor. - * @member {number} vendor - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.vendor = 0 - - /** - * Text version. - * @member {number} version - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.version = 0 - - /** - * Text seqnum. - * @member {number} seqnum - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.seqnum = 0 - - /** - * Text uid. - * @member {number|Long} uid - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.uid = $util.Long ? $util.Long.fromBits(0, 0, false) : 0 - - /** - * Text flag. - * @member {number} flag - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.flag = 0 - - /** - * Text time. - * @member {number|Long} time - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.time = $util.Long ? $util.Long.fromBits(0, 0, false) : 0 - - /** - * Text lang. - * @member {number} lang - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.lang = 0 - - /** - * Text starttime. - * @member {number} starttime - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.starttime = 0 - - /** - * Text offtime. - * @member {number} offtime - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.offtime = 0 - - /** - * Text words. - * @member {Array.} words - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.words = $util.emptyArray - - /** - * Text endOfSegment. - * @member {boolean} endOfSegment - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.endOfSegment = false - - /** - * Text durationMs. - * @member {number} durationMs - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.durationMs = 0 - - /** - * Text dataType. - * @member {string} dataType - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.dataType = "" - - /** - * Text trans. - * @member {Array.} trans - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.trans = $util.emptyArray - - /** - * Text culture. - * @member {string} culture - * @memberof Agora.SpeechToText.Text - * @instance - */ - Text.prototype.culture = "" - - /** - * Creates a new Text instance using the specified properties. - * @function create - * @memberof Agora.SpeechToText.Text - * @static - * @param {Agora.SpeechToText.IText=} [properties] Properties to set - * @returns {Agora.SpeechToText.Text} Text instance - */ - Text.create = function create(properties) { - return new Text(properties) - } - - /** - * Encodes the specified Text message. Does not implicitly {@link Agora.SpeechToText.Text.verify|verify} messages. - * @function encode - * @memberof Agora.SpeechToText.Text - * @static - * @param {Agora.SpeechToText.IText} message Text message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Text.encode = function encode(message, writer) { - if (!writer) writer = $Writer.create() - if (message.vendor != null && Object.hasOwnProperty.call(message, "vendor")) - writer.uint32(/* id 1, wireType 0 = */ 8).int32(message.vendor) - if (message.version != null && Object.hasOwnProperty.call(message, "version")) - writer.uint32(/* id 2, wireType 0 = */ 16).int32(message.version) - if (message.seqnum != null && Object.hasOwnProperty.call(message, "seqnum")) - writer.uint32(/* id 3, wireType 0 = */ 24).int32(message.seqnum) - if (message.uid != null && Object.hasOwnProperty.call(message, "uid")) - writer.uint32(/* id 4, wireType 0 = */ 32).int64(message.uid) - if (message.flag != null && Object.hasOwnProperty.call(message, "flag")) - writer.uint32(/* id 5, wireType 0 = */ 40).int32(message.flag) - if (message.time != null && Object.hasOwnProperty.call(message, "time")) - writer.uint32(/* id 6, wireType 0 = */ 48).int64(message.time) - if (message.lang != null && Object.hasOwnProperty.call(message, "lang")) - writer.uint32(/* id 7, wireType 0 = */ 56).int32(message.lang) - if (message.starttime != null && Object.hasOwnProperty.call(message, "starttime")) - writer.uint32(/* id 8, wireType 0 = */ 64).int32(message.starttime) - if (message.offtime != null && Object.hasOwnProperty.call(message, "offtime")) - writer.uint32(/* id 9, wireType 0 = */ 72).int32(message.offtime) - if (message.words != null && message.words.length) - for (let i = 0; i < message.words.length; ++i) - $root.Agora.SpeechToText.Word.encode( - message.words[i], - writer.uint32(/* id 10, wireType 2 = */ 82).fork(), - ).ldelim() - if (message.endOfSegment != null && Object.hasOwnProperty.call(message, "endOfSegment")) - writer.uint32(/* id 11, wireType 0 = */ 88).bool(message.endOfSegment) - if (message.durationMs != null && Object.hasOwnProperty.call(message, "durationMs")) - writer.uint32(/* id 12, wireType 0 = */ 96).int32(message.durationMs) - if (message.dataType != null && Object.hasOwnProperty.call(message, "dataType")) - writer.uint32(/* id 13, wireType 2 = */ 106).string(message.dataType) - if (message.trans != null && message.trans.length) - for (let i = 0; i < message.trans.length; ++i) - $root.Agora.SpeechToText.Translation.encode( - message.trans[i], - writer.uint32(/* id 14, wireType 2 = */ 114).fork(), - ).ldelim() - if (message.culture != null && Object.hasOwnProperty.call(message, "culture")) - writer.uint32(/* id 15, wireType 2 = */ 122).string(message.culture) - return writer - } - - /** - * Encodes the specified Text message, length delimited. Does not implicitly {@link Agora.SpeechToText.Text.verify|verify} messages. - * @function encodeDelimited - * @memberof Agora.SpeechToText.Text - * @static - * @param {Agora.SpeechToText.IText} message Text message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Text.encodeDelimited = function encodeDelimited(message, writer) { - return this.encode(message, writer).ldelim() - } - - /** - * Decodes a Text message from the specified reader or buffer. - * @function decode - * @memberof Agora.SpeechToText.Text - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @param {number} [length] Message length if known beforehand - * @returns {Agora.SpeechToText.Text} Text - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Text.decode = function decode(reader, length) { - if (!(reader instanceof $Reader)) reader = $Reader.create(reader) - const end = length === undefined ? reader.len : reader.pos + length - const message = new $root.Agora.SpeechToText.Text() - while (reader.pos < end) { - const tag = reader.uint32() - switch (tag >>> 3) { - case 1: { - message.vendor = reader.int32() - break - } - case 2: { - message.version = reader.int32() - break - } - case 3: { - message.seqnum = reader.int32() - break - } - case 4: { - message.uid = reader.int64() - break - } - case 5: { - message.flag = reader.int32() - break - } - case 6: { - message.time = reader.int64() - break - } - case 7: { - message.lang = reader.int32() - break - } - case 8: { - message.starttime = reader.int32() - break - } - case 9: { - message.offtime = reader.int32() - break - } - case 10: { - if (!(message.words && message.words.length)) message.words = [] - message.words.push($root.Agora.SpeechToText.Word.decode(reader, reader.uint32())) - break - } - case 11: { - message.endOfSegment = reader.bool() - break - } - case 12: { - message.durationMs = reader.int32() - break - } - case 13: { - message.dataType = reader.string() - break - } - case 14: { - if (!(message.trans && message.trans.length)) message.trans = [] - message.trans.push( - $root.Agora.SpeechToText.Translation.decode(reader, reader.uint32()), - ) - break - } - case 15: { - message.culture = reader.string() - break - } - default: - reader.skipType(tag & 7) - break - } - } - return message - } - - /** - * Decodes a Text message from the specified reader or buffer, length delimited. - * @function decodeDelimited - * @memberof Agora.SpeechToText.Text - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @returns {Agora.SpeechToText.Text} Text - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Text.decodeDelimited = function decodeDelimited(reader) { - if (!(reader instanceof $Reader)) reader = new $Reader(reader) - return this.decode(reader, reader.uint32()) - } - - /** - * Verifies a Text message. - * @function verify - * @memberof Agora.SpeechToText.Text - * @static - * @param {Object.} message Plain object to verify - * @returns {string|null} `null` if valid, otherwise the reason why it is not - */ - Text.verify = function verify(message) { - if (typeof message !== "object" || message === null) return "object expected" - if (message.vendor != null && message.hasOwnProperty("vendor")) - if (!$util.isInteger(message.vendor)) return "vendor: integer expected" - if (message.version != null && message.hasOwnProperty("version")) - if (!$util.isInteger(message.version)) return "version: integer expected" - if (message.seqnum != null && message.hasOwnProperty("seqnum")) - if (!$util.isInteger(message.seqnum)) return "seqnum: integer expected" - if (message.uid != null && message.hasOwnProperty("uid")) - if ( - !$util.isInteger(message.uid) && - !(message.uid && $util.isInteger(message.uid.low) && $util.isInteger(message.uid.high)) - ) - return "uid: integer|Long expected" - if (message.flag != null && message.hasOwnProperty("flag")) - if (!$util.isInteger(message.flag)) return "flag: integer expected" - if (message.time != null && message.hasOwnProperty("time")) - if ( - !$util.isInteger(message.time) && - !( - message.time && - $util.isInteger(message.time.low) && - $util.isInteger(message.time.high) - ) - ) - return "time: integer|Long expected" - if (message.lang != null && message.hasOwnProperty("lang")) - if (!$util.isInteger(message.lang)) return "lang: integer expected" - if (message.starttime != null && message.hasOwnProperty("starttime")) - if (!$util.isInteger(message.starttime)) return "starttime: integer expected" - if (message.offtime != null && message.hasOwnProperty("offtime")) - if (!$util.isInteger(message.offtime)) return "offtime: integer expected" - if (message.words != null && message.hasOwnProperty("words")) { - if (!Array.isArray(message.words)) return "words: array expected" - for (let i = 0; i < message.words.length; ++i) { - const error = $root.Agora.SpeechToText.Word.verify(message.words[i]) - if (error) return "words." + error - } - } - if (message.endOfSegment != null && message.hasOwnProperty("endOfSegment")) - if (typeof message.endOfSegment !== "boolean") return "endOfSegment: boolean expected" - if (message.durationMs != null && message.hasOwnProperty("durationMs")) - if (!$util.isInteger(message.durationMs)) return "durationMs: integer expected" - if (message.dataType != null && message.hasOwnProperty("dataType")) - if (!$util.isString(message.dataType)) return "dataType: string expected" - if (message.trans != null && message.hasOwnProperty("trans")) { - if (!Array.isArray(message.trans)) return "trans: array expected" - for (let i = 0; i < message.trans.length; ++i) { - const error = $root.Agora.SpeechToText.Translation.verify(message.trans[i]) - if (error) return "trans." + error - } - } - if (message.culture != null && message.hasOwnProperty("culture")) - if (!$util.isString(message.culture)) return "culture: string expected" - return null - } - - /** - * Creates a Text message from a plain object. Also converts values to their respective internal types. - * @function fromObject - * @memberof Agora.SpeechToText.Text - * @static - * @param {Object.} object Plain object - * @returns {Agora.SpeechToText.Text} Text - */ - Text.fromObject = function fromObject(object) { - if (object instanceof $root.Agora.SpeechToText.Text) return object - const message = new $root.Agora.SpeechToText.Text() - if (object.vendor != null) message.vendor = object.vendor | 0 - if (object.version != null) message.version = object.version | 0 - if (object.seqnum != null) message.seqnum = object.seqnum | 0 - if (object.uid != null) - if ($util.Long) (message.uid = $util.Long.fromValue(object.uid)).unsigned = false - else if (typeof object.uid === "string") message.uid = parseInt(object.uid, 10) - else if (typeof object.uid === "number") message.uid = object.uid - else if (typeof object.uid === "object") - message.uid = new $util.LongBits(object.uid.low >>> 0, object.uid.high >>> 0).toNumber() - if (object.flag != null) message.flag = object.flag | 0 - if (object.time != null) - if ($util.Long) (message.time = $util.Long.fromValue(object.time)).unsigned = false - else if (typeof object.time === "string") message.time = parseInt(object.time, 10) - else if (typeof object.time === "number") message.time = object.time - else if (typeof object.time === "object") - message.time = new $util.LongBits( - object.time.low >>> 0, - object.time.high >>> 0, - ).toNumber() - if (object.lang != null) message.lang = object.lang | 0 - if (object.starttime != null) message.starttime = object.starttime | 0 - if (object.offtime != null) message.offtime = object.offtime | 0 - if (object.words) { - if (!Array.isArray(object.words)) - throw TypeError(".Agora.SpeechToText.Text.words: array expected") - message.words = [] - for (let i = 0; i < object.words.length; ++i) { - if (typeof object.words[i] !== "object") - throw TypeError(".Agora.SpeechToText.Text.words: object expected") - message.words[i] = $root.Agora.SpeechToText.Word.fromObject(object.words[i]) - } - } - if (object.endOfSegment != null) message.endOfSegment = Boolean(object.endOfSegment) - if (object.durationMs != null) message.durationMs = object.durationMs | 0 - if (object.dataType != null) message.dataType = String(object.dataType) - if (object.trans) { - if (!Array.isArray(object.trans)) - throw TypeError(".Agora.SpeechToText.Text.trans: array expected") - message.trans = [] - for (let i = 0; i < object.trans.length; ++i) { - if (typeof object.trans[i] !== "object") - throw TypeError(".Agora.SpeechToText.Text.trans: object expected") - message.trans[i] = $root.Agora.SpeechToText.Translation.fromObject(object.trans[i]) - } - } - if (object.culture != null) message.culture = String(object.culture) - return message - } - - /** - * Creates a plain object from a Text message. Also converts values to other types if specified. - * @function toObject - * @memberof Agora.SpeechToText.Text - * @static - * @param {Agora.SpeechToText.Text} message Text - * @param {$protobuf.IConversionOptions} [options] Conversion options - * @returns {Object.} Plain object - */ - Text.toObject = function toObject(message, options) { - if (!options) options = {} - const object = {} - if (options.arrays || options.defaults) { - object.words = [] - object.trans = [] - } - if (options.defaults) { - object.vendor = 0 - object.version = 0 - object.seqnum = 0 - if ($util.Long) { - const long = new $util.Long(0, 0, false) - object.uid = - options.longs === String - ? long.toString() - : options.longs === Number - ? long.toNumber() - : long - } else object.uid = options.longs === String ? "0" : 0 - object.flag = 0 - if ($util.Long) { - const long = new $util.Long(0, 0, false) - object.time = - options.longs === String - ? long.toString() - : options.longs === Number - ? long.toNumber() - : long - } else object.time = options.longs === String ? "0" : 0 - object.lang = 0 - object.starttime = 0 - object.offtime = 0 - object.endOfSegment = false - object.durationMs = 0 - object.dataType = "" - object.culture = "" - } - if (message.vendor != null && message.hasOwnProperty("vendor")) - object.vendor = message.vendor - if (message.version != null && message.hasOwnProperty("version")) - object.version = message.version - if (message.seqnum != null && message.hasOwnProperty("seqnum")) - object.seqnum = message.seqnum - if (message.uid != null && message.hasOwnProperty("uid")) - if (typeof message.uid === "number") - object.uid = options.longs === String ? String(message.uid) : message.uid - else - object.uid = - options.longs === String - ? $util.Long.prototype.toString.call(message.uid) - : options.longs === Number - ? new $util.LongBits(message.uid.low >>> 0, message.uid.high >>> 0).toNumber() - : message.uid - if (message.flag != null && message.hasOwnProperty("flag")) object.flag = message.flag - if (message.time != null && message.hasOwnProperty("time")) - if (typeof message.time === "number") - object.time = options.longs === String ? String(message.time) : message.time - else - object.time = - options.longs === String - ? $util.Long.prototype.toString.call(message.time) - : options.longs === Number - ? new $util.LongBits(message.time.low >>> 0, message.time.high >>> 0).toNumber() - : message.time - if (message.lang != null && message.hasOwnProperty("lang")) object.lang = message.lang - if (message.starttime != null && message.hasOwnProperty("starttime")) - object.starttime = message.starttime - if (message.offtime != null && message.hasOwnProperty("offtime")) - object.offtime = message.offtime - if (message.words && message.words.length) { - object.words = [] - for (let j = 0; j < message.words.length; ++j) - object.words[j] = $root.Agora.SpeechToText.Word.toObject(message.words[j], options) - } - if (message.endOfSegment != null && message.hasOwnProperty("endOfSegment")) - object.endOfSegment = message.endOfSegment - if (message.durationMs != null && message.hasOwnProperty("durationMs")) - object.durationMs = message.durationMs - if (message.dataType != null && message.hasOwnProperty("dataType")) - object.dataType = message.dataType - if (message.trans && message.trans.length) { - object.trans = [] - for (let j = 0; j < message.trans.length; ++j) - object.trans[j] = $root.Agora.SpeechToText.Translation.toObject( - message.trans[j], - options, - ) - } - if (message.culture != null && message.hasOwnProperty("culture")) - object.culture = message.culture - return object - } - - /** - * Converts this Text to JSON. - * @function toJSON - * @memberof Agora.SpeechToText.Text - * @instance - * @returns {Object.} JSON object - */ - Text.prototype.toJSON = function toJSON() { - return this.constructor.toObject(this, $protobuf.util.toJSONOptions) - } - - /** - * Gets the default type url for Text - * @function getTypeUrl - * @memberof Agora.SpeechToText.Text - * @static - * @param {string} [typeUrlPrefix] your custom typeUrlPrefix(default "type.googleapis.com") - * @returns {string} The default type url - */ - Text.getTypeUrl = function getTypeUrl(typeUrlPrefix) { - if (typeUrlPrefix === undefined) { - typeUrlPrefix = "type.googleapis.com" - } - return typeUrlPrefix + "/Agora.SpeechToText.Text" - } - - return Text - })() - - SpeechToText.Word = (function () { - /** - * Properties of a Word. - * @memberof Agora.SpeechToText - * @interface IWord - * @property {string|null} [text] Word text - * @property {number|null} [startMs] Word startMs - * @property {number|null} [durationMs] Word durationMs - * @property {boolean|null} [isFinal] Word isFinal - * @property {number|null} [confidence] Word confidence - */ - - /** - * Constructs a new Word. - * @memberof Agora.SpeechToText - * @classdesc Represents a Word. - * @implements IWord - * @constructor - * @param {Agora.SpeechToText.IWord=} [properties] Properties to set - */ - function Word(properties) { - if (properties) - for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i) - if (properties[keys[i]] != null) this[keys[i]] = properties[keys[i]] - } - - /** - * Word text. - * @member {string} text - * @memberof Agora.SpeechToText.Word - * @instance - */ - Word.prototype.text = "" - - /** - * Word startMs. - * @member {number} startMs - * @memberof Agora.SpeechToText.Word - * @instance - */ - Word.prototype.startMs = 0 - - /** - * Word durationMs. - * @member {number} durationMs - * @memberof Agora.SpeechToText.Word - * @instance - */ - Word.prototype.durationMs = 0 - - /** - * Word isFinal. - * @member {boolean} isFinal - * @memberof Agora.SpeechToText.Word - * @instance - */ - Word.prototype.isFinal = false - - /** - * Word confidence. - * @member {number} confidence - * @memberof Agora.SpeechToText.Word - * @instance - */ - Word.prototype.confidence = 0 - - /** - * Creates a new Word instance using the specified properties. - * @function create - * @memberof Agora.SpeechToText.Word - * @static - * @param {Agora.SpeechToText.IWord=} [properties] Properties to set - * @returns {Agora.SpeechToText.Word} Word instance - */ - Word.create = function create(properties) { - return new Word(properties) - } - - /** - * Encodes the specified Word message. Does not implicitly {@link Agora.SpeechToText.Word.verify|verify} messages. - * @function encode - * @memberof Agora.SpeechToText.Word - * @static - * @param {Agora.SpeechToText.IWord} message Word message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Word.encode = function encode(message, writer) { - if (!writer) writer = $Writer.create() - if (message.text != null && Object.hasOwnProperty.call(message, "text")) - writer.uint32(/* id 1, wireType 2 = */ 10).string(message.text) - if (message.startMs != null && Object.hasOwnProperty.call(message, "startMs")) - writer.uint32(/* id 2, wireType 0 = */ 16).int32(message.startMs) - if (message.durationMs != null && Object.hasOwnProperty.call(message, "durationMs")) - writer.uint32(/* id 3, wireType 0 = */ 24).int32(message.durationMs) - if (message.isFinal != null && Object.hasOwnProperty.call(message, "isFinal")) - writer.uint32(/* id 4, wireType 0 = */ 32).bool(message.isFinal) - if (message.confidence != null && Object.hasOwnProperty.call(message, "confidence")) - writer.uint32(/* id 5, wireType 1 = */ 41).double(message.confidence) - return writer - } - - /** - * Encodes the specified Word message, length delimited. Does not implicitly {@link Agora.SpeechToText.Word.verify|verify} messages. - * @function encodeDelimited - * @memberof Agora.SpeechToText.Word - * @static - * @param {Agora.SpeechToText.IWord} message Word message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Word.encodeDelimited = function encodeDelimited(message, writer) { - return this.encode(message, writer).ldelim() - } - - /** - * Decodes a Word message from the specified reader or buffer. - * @function decode - * @memberof Agora.SpeechToText.Word - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @param {number} [length] Message length if known beforehand - * @returns {Agora.SpeechToText.Word} Word - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Word.decode = function decode(reader, length) { - if (!(reader instanceof $Reader)) reader = $Reader.create(reader) - const end = length === undefined ? reader.len : reader.pos + length - const message = new $root.Agora.SpeechToText.Word() - while (reader.pos < end) { - const tag = reader.uint32() - switch (tag >>> 3) { - case 1: { - message.text = reader.string() - break - } - case 2: { - message.startMs = reader.int32() - break - } - case 3: { - message.durationMs = reader.int32() - break - } - case 4: { - message.isFinal = reader.bool() - break - } - case 5: { - message.confidence = reader.double() - break - } - default: - reader.skipType(tag & 7) - break - } - } - return message - } - - /** - * Decodes a Word message from the specified reader or buffer, length delimited. - * @function decodeDelimited - * @memberof Agora.SpeechToText.Word - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @returns {Agora.SpeechToText.Word} Word - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Word.decodeDelimited = function decodeDelimited(reader) { - if (!(reader instanceof $Reader)) reader = new $Reader(reader) - return this.decode(reader, reader.uint32()) - } - - /** - * Verifies a Word message. - * @function verify - * @memberof Agora.SpeechToText.Word - * @static - * @param {Object.} message Plain object to verify - * @returns {string|null} `null` if valid, otherwise the reason why it is not - */ - Word.verify = function verify(message) { - if (typeof message !== "object" || message === null) return "object expected" - if (message.text != null && message.hasOwnProperty("text")) - if (!$util.isString(message.text)) return "text: string expected" - if (message.startMs != null && message.hasOwnProperty("startMs")) - if (!$util.isInteger(message.startMs)) return "startMs: integer expected" - if (message.durationMs != null && message.hasOwnProperty("durationMs")) - if (!$util.isInteger(message.durationMs)) return "durationMs: integer expected" - if (message.isFinal != null && message.hasOwnProperty("isFinal")) - if (typeof message.isFinal !== "boolean") return "isFinal: boolean expected" - if (message.confidence != null && message.hasOwnProperty("confidence")) - if (typeof message.confidence !== "number") return "confidence: number expected" - return null - } - - /** - * Creates a Word message from a plain object. Also converts values to their respective internal types. - * @function fromObject - * @memberof Agora.SpeechToText.Word - * @static - * @param {Object.} object Plain object - * @returns {Agora.SpeechToText.Word} Word - */ - Word.fromObject = function fromObject(object) { - if (object instanceof $root.Agora.SpeechToText.Word) return object - const message = new $root.Agora.SpeechToText.Word() - if (object.text != null) message.text = String(object.text) - if (object.startMs != null) message.startMs = object.startMs | 0 - if (object.durationMs != null) message.durationMs = object.durationMs | 0 - if (object.isFinal != null) message.isFinal = Boolean(object.isFinal) - if (object.confidence != null) message.confidence = Number(object.confidence) - return message - } - - /** - * Creates a plain object from a Word message. Also converts values to other types if specified. - * @function toObject - * @memberof Agora.SpeechToText.Word - * @static - * @param {Agora.SpeechToText.Word} message Word - * @param {$protobuf.IConversionOptions} [options] Conversion options - * @returns {Object.} Plain object - */ - Word.toObject = function toObject(message, options) { - if (!options) options = {} - const object = {} - if (options.defaults) { - object.text = "" - object.startMs = 0 - object.durationMs = 0 - object.isFinal = false - object.confidence = 0 - } - if (message.text != null && message.hasOwnProperty("text")) object.text = message.text - if (message.startMs != null && message.hasOwnProperty("startMs")) - object.startMs = message.startMs - if (message.durationMs != null && message.hasOwnProperty("durationMs")) - object.durationMs = message.durationMs - if (message.isFinal != null && message.hasOwnProperty("isFinal")) - object.isFinal = message.isFinal - if (message.confidence != null && message.hasOwnProperty("confidence")) - object.confidence = - options.json && !isFinite(message.confidence) - ? String(message.confidence) - : message.confidence - return object - } - - /** - * Converts this Word to JSON. - * @function toJSON - * @memberof Agora.SpeechToText.Word - * @instance - * @returns {Object.} JSON object - */ - Word.prototype.toJSON = function toJSON() { - return this.constructor.toObject(this, $protobuf.util.toJSONOptions) - } - - /** - * Gets the default type url for Word - * @function getTypeUrl - * @memberof Agora.SpeechToText.Word - * @static - * @param {string} [typeUrlPrefix] your custom typeUrlPrefix(default "type.googleapis.com") - * @returns {string} The default type url - */ - Word.getTypeUrl = function getTypeUrl(typeUrlPrefix) { - if (typeUrlPrefix === undefined) { - typeUrlPrefix = "type.googleapis.com" - } - return typeUrlPrefix + "/Agora.SpeechToText.Word" - } - - return Word - })() - - SpeechToText.Translation = (function () { - /** - * Properties of a Translation. - * @memberof Agora.SpeechToText - * @interface ITranslation - * @property {boolean|null} [isFinal] Translation isFinal - * @property {string|null} [lang] Translation lang - * @property {Array.|null} [texts] Translation texts - */ - - /** - * Constructs a new Translation. - * @memberof Agora.SpeechToText - * @classdesc Represents a Translation. - * @implements ITranslation - * @constructor - * @param {Agora.SpeechToText.ITranslation=} [properties] Properties to set - */ - function Translation(properties) { - this.texts = [] - if (properties) - for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i) - if (properties[keys[i]] != null) this[keys[i]] = properties[keys[i]] - } - - /** - * Translation isFinal. - * @member {boolean} isFinal - * @memberof Agora.SpeechToText.Translation - * @instance - */ - Translation.prototype.isFinal = false - - /** - * Translation lang. - * @member {string} lang - * @memberof Agora.SpeechToText.Translation - * @instance - */ - Translation.prototype.lang = "" - - /** - * Translation texts. - * @member {Array.} texts - * @memberof Agora.SpeechToText.Translation - * @instance - */ - Translation.prototype.texts = $util.emptyArray - - /** - * Creates a new Translation instance using the specified properties. - * @function create - * @memberof Agora.SpeechToText.Translation - * @static - * @param {Agora.SpeechToText.ITranslation=} [properties] Properties to set - * @returns {Agora.SpeechToText.Translation} Translation instance - */ - Translation.create = function create(properties) { - return new Translation(properties) - } - - /** - * Encodes the specified Translation message. Does not implicitly {@link Agora.SpeechToText.Translation.verify|verify} messages. - * @function encode - * @memberof Agora.SpeechToText.Translation - * @static - * @param {Agora.SpeechToText.ITranslation} message Translation message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Translation.encode = function encode(message, writer) { - if (!writer) writer = $Writer.create() - if (message.isFinal != null && Object.hasOwnProperty.call(message, "isFinal")) - writer.uint32(/* id 1, wireType 0 = */ 8).bool(message.isFinal) - if (message.lang != null && Object.hasOwnProperty.call(message, "lang")) - writer.uint32(/* id 2, wireType 2 = */ 18).string(message.lang) - if (message.texts != null && message.texts.length) - for (let i = 0; i < message.texts.length; ++i) - writer.uint32(/* id 3, wireType 2 = */ 26).string(message.texts[i]) - return writer - } - - /** - * Encodes the specified Translation message, length delimited. Does not implicitly {@link Agora.SpeechToText.Translation.verify|verify} messages. - * @function encodeDelimited - * @memberof Agora.SpeechToText.Translation - * @static - * @param {Agora.SpeechToText.ITranslation} message Translation message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Translation.encodeDelimited = function encodeDelimited(message, writer) { - return this.encode(message, writer).ldelim() - } - - /** - * Decodes a Translation message from the specified reader or buffer. - * @function decode - * @memberof Agora.SpeechToText.Translation - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @param {number} [length] Message length if known beforehand - * @returns {Agora.SpeechToText.Translation} Translation - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Translation.decode = function decode(reader, length) { - if (!(reader instanceof $Reader)) reader = $Reader.create(reader) - const end = length === undefined ? reader.len : reader.pos + length - const message = new $root.Agora.SpeechToText.Translation() - while (reader.pos < end) { - const tag = reader.uint32() - switch (tag >>> 3) { - case 1: { - message.isFinal = reader.bool() - break - } - case 2: { - message.lang = reader.string() - break - } - case 3: { - if (!(message.texts && message.texts.length)) message.texts = [] - message.texts.push(reader.string()) - break - } - default: - reader.skipType(tag & 7) - break - } - } - return message - } - - /** - * Decodes a Translation message from the specified reader or buffer, length delimited. - * @function decodeDelimited - * @memberof Agora.SpeechToText.Translation - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @returns {Agora.SpeechToText.Translation} Translation - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Translation.decodeDelimited = function decodeDelimited(reader) { - if (!(reader instanceof $Reader)) reader = new $Reader(reader) - return this.decode(reader, reader.uint32()) - } - - /** - * Verifies a Translation message. - * @function verify - * @memberof Agora.SpeechToText.Translation - * @static - * @param {Object.} message Plain object to verify - * @returns {string|null} `null` if valid, otherwise the reason why it is not - */ - Translation.verify = function verify(message) { - if (typeof message !== "object" || message === null) return "object expected" - if (message.isFinal != null && message.hasOwnProperty("isFinal")) - if (typeof message.isFinal !== "boolean") return "isFinal: boolean expected" - if (message.lang != null && message.hasOwnProperty("lang")) - if (!$util.isString(message.lang)) return "lang: string expected" - if (message.texts != null && message.hasOwnProperty("texts")) { - if (!Array.isArray(message.texts)) return "texts: array expected" - for (let i = 0; i < message.texts.length; ++i) - if (!$util.isString(message.texts[i])) return "texts: string[] expected" - } - return null - } - - /** - * Creates a Translation message from a plain object. Also converts values to their respective internal types. - * @function fromObject - * @memberof Agora.SpeechToText.Translation - * @static - * @param {Object.} object Plain object - * @returns {Agora.SpeechToText.Translation} Translation - */ - Translation.fromObject = function fromObject(object) { - if (object instanceof $root.Agora.SpeechToText.Translation) return object - const message = new $root.Agora.SpeechToText.Translation() - if (object.isFinal != null) message.isFinal = Boolean(object.isFinal) - if (object.lang != null) message.lang = String(object.lang) - if (object.texts) { - if (!Array.isArray(object.texts)) - throw TypeError(".Agora.SpeechToText.Translation.texts: array expected") - message.texts = [] - for (let i = 0; i < object.texts.length; ++i) message.texts[i] = String(object.texts[i]) - } - return message - } - - /** - * Creates a plain object from a Translation message. Also converts values to other types if specified. - * @function toObject - * @memberof Agora.SpeechToText.Translation - * @static - * @param {Agora.SpeechToText.Translation} message Translation - * @param {$protobuf.IConversionOptions} [options] Conversion options - * @returns {Object.} Plain object - */ - Translation.toObject = function toObject(message, options) { - if (!options) options = {} - const object = {} - if (options.arrays || options.defaults) object.texts = [] - if (options.defaults) { - object.isFinal = false - object.lang = "" - } - if (message.isFinal != null && message.hasOwnProperty("isFinal")) - object.isFinal = message.isFinal - if (message.lang != null && message.hasOwnProperty("lang")) object.lang = message.lang - if (message.texts && message.texts.length) { - object.texts = [] - for (let j = 0; j < message.texts.length; ++j) object.texts[j] = message.texts[j] - } - return object - } - - /** - * Converts this Translation to JSON. - * @function toJSON - * @memberof Agora.SpeechToText.Translation - * @instance - * @returns {Object.} JSON object - */ - Translation.prototype.toJSON = function toJSON() { - return this.constructor.toObject(this, $protobuf.util.toJSONOptions) - } - - /** - * Gets the default type url for Translation - * @function getTypeUrl - * @memberof Agora.SpeechToText.Translation - * @static - * @param {string} [typeUrlPrefix] your custom typeUrlPrefix(default "type.googleapis.com") - * @returns {string} The default type url - */ - Translation.getTypeUrl = function getTypeUrl(typeUrlPrefix) { - if (typeUrlPrefix === undefined) { - typeUrlPrefix = "type.googleapis.com" - } - return typeUrlPrefix + "/Agora.SpeechToText.Translation" - } - - return Translation - })() - - return SpeechToText - })() - - return Agora -})()) - -export { $root as default } diff --git a/playground/src/store/reducers/global.ts b/playground/src/store/reducers/global.ts index 377ece81..995b22d1 100644 --- a/playground/src/store/reducers/global.ts +++ b/playground/src/store/reducers/global.ts @@ -1,4 +1,4 @@ -import { IOptions, IChatItem } from "@/types" +import { IOptions, IChatItem, Language, VoiceType } from "@/types" import { createSlice, PayloadAction } from "@reduxjs/toolkit" import { DEFAULT_OPTIONS, COLOR_LIST, setOptionsToLocal, genRandomChatList } from "@/common" @@ -6,7 +6,9 @@ export interface InitialState { options: IOptions roomConnected: boolean, agentConnected: boolean, - themeColor: string + themeColor: string, + language: Language + voiceType: VoiceType chatItems: IChatItem[] } @@ -16,6 +18,8 @@ const getInitialState = (): InitialState => { themeColor: COLOR_LIST[0].active, roomConnected: false, agentConnected: false, + language: "en-US", + voiceType: "female", chatItems: [] } } @@ -75,6 +79,12 @@ export const globalSlice = createSlice({ setAgentConnected: (state, action: PayloadAction) => { state.agentConnected = action.payload }, + setLanguage: (state, action: PayloadAction) => { + state.language = action.payload + }, + setVoiceType: (state, action: PayloadAction) => { + state.voiceType = action.payload + }, reset: (state) => { Object.assign(state, getInitialState()) document.documentElement.style.setProperty('--theme-color', COLOR_LIST[0].active); @@ -82,7 +92,9 @@ export const globalSlice = createSlice({ }, }) -export const { reset, setOptions, setRoomConnected, setAgentConnected, addChatItem, setThemeColor } = +export const { reset, setOptions, + setRoomConnected, setAgentConnected, setVoiceType, + addChatItem, setThemeColor, setLanguage } = globalSlice.actions export default globalSlice.reducer diff --git a/playground/src/types/index.ts b/playground/src/types/index.ts index a42188d7..1f695c39 100644 --- a/playground/src/types/index.ts +++ b/playground/src/types/index.ts @@ -1,3 +1,6 @@ +export type Language = "en-US" | "zh-CN" +export type VoiceType = "male" | "female" + export interface ColorItem { active: string, default: string @@ -30,3 +33,27 @@ export interface ITextItem { isFinal: boolean } + +export interface LanguageOptionItem { + label: string + value: Language +} + + +export interface VoiceOptionItem { + label: string + value: VoiceType +} + + +export interface OptionType { + value: string; + label: string; +} + + +export interface IPdfData { + fileName: string, + collection: string +} + From 09d4b6ae0dafe4c9fb21914f23a11eb402de1857 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Tue, 13 Aug 2024 20:39:59 -0700 Subject: [PATCH 19/34] feat: update agora_rtc --- agents/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/manifest.json b/agents/manifest.json index 571aaf12..298c4af8 100644 --- a/agents/manifest.json +++ b/agents/manifest.json @@ -27,7 +27,7 @@ { "type": "extension", "name": "agora_rtc", - "version": "0.4.0" + "version": "0.4.1-rc3" } ] } \ No newline at end of file From 8d428b22c596ce1ee53507fd12201b23ca4b5e55 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Tue, 13 Aug 2024 20:40:52 -0700 Subject: [PATCH 20/34] fix: audio stuck, calculate ttfb --- .../extension/cosy_tts/cosy_tts_extension.py | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/agents/addon/extension/cosy_tts/cosy_tts_extension.py b/agents/addon/extension/cosy_tts/cosy_tts_extension.py index cd57d406..07c15bf4 100644 --- a/agents/addon/extension/cosy_tts/cosy_tts_extension.py +++ b/agents/addon/extension/cosy_tts/cosy_tts_extension.py @@ -32,6 +32,7 @@ def __init__(self, rte: RteEnv, sample_rate: int, need_interrupt_callback): self.sample_rate = sample_rate self.frame_size = int(self.sample_rate * 1 * 2 / 100) self.ts = datetime.now() + self.ttfb = None # time to first byte self.need_interrupt_callback = need_interrupt_callback self.closed = False @@ -40,6 +41,7 @@ def need_interrupt(self) -> bool: def set_input_ts(self, ts: datetime): self.ts = ts + self.ttfb = None # clear to calculate again def on_open(self): logger.info("websocket is open.") @@ -65,36 +67,24 @@ def get_frame(self, data: bytes) -> PcmFrame: f.set_number_of_channels(1) # f.set_timestamp = 0 f.set_data_fmt(PcmFrameDataFmt.INTERLEAVE) - f.set_samples_per_channel(self.sample_rate // 100) - f.alloc_buf(self.frame_size) + f.set_samples_per_channel(len(data) // 2) + f.alloc_buf(len(data)) buff = f.lock_buf() - if len(data) < self.frame_size: - buff[:] = bytes(self.frame_size) # fill with 0 - buff[: len(data)] = data + buff[:] = data f.unlock_buf(buff) return f def on_data(self, data: bytes) -> None: if self.need_interrupt(): return + if self.ttfb is None: + self.ttfb = datetime.now() - self.ts + logger.info("TTS TTFB {}ms".format(int(self.ttfb.total_seconds() * 1000))) # logger.info("audio result length: %d, %d", len(data), self.frame_size) try: - chunk = int(len(data) / self.frame_size) - offset = 0 - for i in range(0, chunk): - if self.need_interrupt(): - return - f = self.get_frame(data[offset : offset + self.frame_size]) - self.rte.send_pcm_frame(f) - offset += self.frame_size - - if self.need_interrupt(): - return - if offset < len(data): - size = len(data) - offset - f = self.get_frame(data[offset : offset + size]) - self.rte.send_pcm_frame(f) + f = self.get_frame(data) + self.rte.send_pcm_frame(f) except Exception as e: logger.exception(e) From ff31405396b9bf5740a5b7d3a95db1e8a99189e7 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Tue, 13 Aug 2024 20:43:46 -0700 Subject: [PATCH 21/34] fix: too slow output --- .../qwen_llm_python/qwen_llm_extension.py | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py b/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py index 2094d4e8..bdc2e0cc 100644 --- a/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py +++ b/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py @@ -19,24 +19,11 @@ import json from datetime import datetime import threading +import re from http import HTTPStatus from .log import logger -def isEnd(content: str) -> bool: - last = content[len(content) - 1] - return ( - last == "," - or last == "，" - or last == "." - or last == "。" - or last == "?" - or last == "？" - or last == "!" - or last == "！" - ) - - class QWenLLMExtension(Extension): def __init__(self, name: str): super().__init__(name) @@ -48,6 +35,7 @@ def __init__(self, name: str): self.stopped = False self.thread = None self.outdate_ts = datetime.now() + self.sentence_expr = re.compile(r".+?[,，.。!！?？:：]", re.DOTALL) self.queue = queue.Queue() self.mutex = threading.Lock() @@ -145,7 +133,6 @@ def stream_chat(self, ts: datetime.time, messages: List[Any], callback): for response in responses: if self.need_interrupt(ts): logger.warning("out of date, %s, %s", self.outdate_ts, ts) - total += partial partial = "" # discard not sent break if response.status_code == HTTPStatus.OK: @@ -153,11 +140,15 @@ def stream_chat(self, ts: datetime.time, messages: List[Any], callback): if len(temp) == 0: continue partial += temp - if (isEnd(temp) and len(partial) > 10) or len(partial) > 50: + total += temp + + m = self.sentence_expr.match(partial) + if m is not None: + sentence = m.group(0) + partial = partial[m.end(0) :] if callback is not None: - callback(partial, False) - total += partial - partial = "" + callback(sentence, False) + else: logger.warning( "request_id: {}, status_code: {}, error code: {}, error message: {}".format( @@ -169,12 +160,9 @@ def stream_chat(self, ts: datetime.time, messages: List[Any], callback): ) break - if len(total) > 0 or len(partial) > 0: # make sure no empty answer - if callback is not None: - callback(partial, True) - if len(partial) > 0: - total += partial - partial = "" + # always send end_of_segment + if callback is not None: + callback(partial, True) logger.info("stream_chat full_answer {}".format(total)) return total From 423047595b7c95b5a13f4773cc48de65d5c1493b Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Wed, 14 Aug 2024 11:51:00 +0800 Subject: [PATCH 22/34] feat: fix text glitches --- playground/src/components/icons/network/index.tsx | 2 -- playground/src/platform/pc/description/index.tsx | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/playground/src/components/icons/network/index.tsx b/playground/src/components/icons/network/index.tsx index 332d66a9..1950cda7 100644 --- a/playground/src/components/icons/network/index.tsx +++ b/playground/src/components/icons/network/index.tsx @@ -1,5 +1,3 @@ -// https://doc.shengwang.cn/api-ref/rtc/javascript/interfaces/networkquality - import averageSvg from "@/assets/network/average.svg" import goodSvg from "@/assets/network/good.svg" import poorSvg from "@/assets/network/poor.svg" diff --git a/playground/src/platform/pc/description/index.tsx b/playground/src/platform/pc/description/index.tsx index f7965392..1e679c79 100644 --- a/playground/src/platform/pc/description/index.tsx +++ b/playground/src/platform/pc/description/index.tsx @@ -85,7 +85,7 @@ const Description = () => { return
Description - RTT is an AI voice assistant powered by Shengwang Agents, Deepgram, Eleven Labs, and ChatGPT. It is running on Shengwang Playground. + Astra is an AI voice assistant powered by T.E.N framework, Deepgram, Eleven Labs, and ChatGPT. {!agentConnected ? "Connect" : "Disconnect"} From de5d79cabd7a465e803071b54982855d5284b6ab Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Wed, 14 Aug 2024 14:22:42 +0800 Subject: [PATCH 23/34] feat: support graph selection --- docker-compose.yml | 10 ++++++++++ playground/src/common/constant.ts | 16 +++++++++++++++- playground/src/common/request.ts | 6 ++++-- .../src/platform/pc/chat/index.module.scss | 8 +++++++- playground/src/platform/pc/chat/index.tsx | 14 +++++++++++--- playground/src/platform/pc/description/index.tsx | 4 +++- playground/src/store/reducers/global.ts | 11 ++++++++--- playground/src/types/index.ts | 4 ++++ 8 files changed, 62 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 179790ba..5c81a871 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,16 @@ services: restart: always ports: - "3000:3000" + + astra_playground_dev: + image: node:20-alpine + container_name: astra_playground_dev + restart: always + command: sh -c "cd /app/playground && npm i && npm run dev" #build && npm run start" + ports: + - "3002:3000" + volumes: + - ./:/app astra_graph_designer: image: agoraio/astra_graph_designer:0.3.0 container_name: astra_graph_designer diff --git a/playground/src/common/constant.ts b/playground/src/common/constant.ts index e520a195..2602e7b1 100644 --- a/playground/src/common/constant.ts +++ b/playground/src/common/constant.ts @@ -1,4 +1,4 @@ -import { IOptions, ColorItem, LanguageOptionItem, VoiceOptionItem } from "@/types" +import { IOptions, ColorItem, LanguageOptionItem, VoiceOptionItem, GraphNameOptionItem, GraphOptionItem } from "@/types" export const REQUEST_URL = process.env.NEXT_PUBLIC_REQUEST_URL ?? "" export const GITHUB_URL = "https://github.com/rte-design/ASTRA.ai" @@ -19,6 +19,20 @@ export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [ value: "zh-CN" } ] +export const GRAPH_OPTIONS: GraphOptionItem[] = [ + { + label: "Voice Agent - OpenAI LLM + Azure TTS", + value: "va.openai.azure" + }, + { + label: "Voice Agent with Vision - OpenAI LLM + Azure TTS", + value: "camera.va.openai.azure" + }, + { + label: "Voice Agent with Knowledge - RAG + Qwen LLM + Cosy TTS", + value: "va.qwen.rag" + }, +] export const VOICE_OPTIONS: VoiceOptionItem[] = [ { label: "Male", diff --git a/playground/src/common/request.ts b/playground/src/common/request.ts index 8329911d..6d6a6692 100644 --- a/playground/src/common/request.ts +++ b/playground/src/common/request.ts @@ -6,6 +6,7 @@ interface StartRequestConfig { userId: number, language: string voiceType: string + graphName: string } interface GenAgoraDataConfig { @@ -34,14 +35,15 @@ export const apiGenAgoraData = async (config: GenAgoraDataConfig) => { export const apiStartService = async (config: StartRequestConfig): Promise => { const url = `${REQUEST_URL}/start` - const { language, channel, userId, voiceType } = config + const { language, channel, userId, voiceType, graphName } = config const data = { request_id: genUUID(), agora_asr_language: language, channel_name: channel, openai_proxy_url: "", remote_stream_id: userId, - voice_type: voiceType + voice_type: voiceType, + graph_name: graphName, } let resp: any = await fetch(url, { method: "POST", diff --git a/playground/src/platform/pc/chat/index.module.scss b/playground/src/platform/pc/chat/index.module.scss index b5c2707c..ecc565d7 100644 --- a/playground/src/platform/pc/chat/index.module.scss +++ b/playground/src/platform/pc/chat/index.module.scss @@ -20,6 +20,9 @@ .left { flex: 1 1 auto; + display: flex; + align-items: center; + gap: 5px; .text { margin-left: 4px; @@ -32,14 +35,17 @@ } .languageSelect { - margin-left: 12px; width: 100px; } } .right { + display: flex; + align-items: center; + gap: 5px; flex: 0 0 230px; + justify-content: right; } } diff --git a/playground/src/platform/pc/chat/index.tsx b/playground/src/platform/pc/chat/index.tsx index 98cdd083..1e218aa2 100644 --- a/playground/src/platform/pc/chat/index.tsx +++ b/playground/src/platform/pc/chat/index.tsx @@ -5,8 +5,9 @@ import ChatItem from "./chatItem" import { genRandomChatList, useAppDispatch, useAutoScroll, LANGUAGE_OPTIONS, useAppSelector, + GRAPH_OPTIONS, } from "@/common" -import { setLanguage } from "@/store/reducers/global" +import { setGraphName, setLanguage } from "@/store/reducers/global" import { Select, } from 'antd'; import PdfSelect from "@/components/pdfSelect" @@ -19,6 +20,7 @@ const Chat = () => { const dispatch = useAppDispatch() const chatItems = useAppSelector(state => state.global.chatItems) const language = useAppSelector(state => state.global.language) + const graphName = useAppSelector(state => state.global.graphName) const agentConnected = useAppSelector(state => state.global.agentConnected) // const chatItems = genRandomChatList(10) @@ -32,13 +34,19 @@ const Chat = () => { dispatch(setLanguage(val)) } - + const onGraphNameChange = (val: any) => { + dispatch(setGraphName(val)) + } return
- Chat + Agent Graph + + Lang diff --git a/playground/src/platform/pc/description/index.tsx b/playground/src/platform/pc/description/index.tsx index 1e679c79..432e2a94 100644 --- a/playground/src/platform/pc/description/index.tsx +++ b/playground/src/platform/pc/description/index.tsx @@ -17,6 +17,7 @@ const Description = () => { const userId = useAppSelector(state => state.global.options.userId) const language = useAppSelector(state => state.global.language) const voiceType = useAppSelector(state => state.global.voiceType) + const graphName = useAppSelector(state => state.global.graphName) const [loading, setLoading] = useState(false) useEffect(() => { @@ -48,7 +49,8 @@ const Description = () => { channel, userId, language, - voiceType + voiceType, + graphName }) const { code, msg } = res || {} if (code != 0) { diff --git a/playground/src/store/reducers/global.ts b/playground/src/store/reducers/global.ts index 995b22d1..dff67e62 100644 --- a/playground/src/store/reducers/global.ts +++ b/playground/src/store/reducers/global.ts @@ -9,7 +9,8 @@ export interface InitialState { themeColor: string, language: Language voiceType: VoiceType - chatItems: IChatItem[] + chatItems: IChatItem[], + graphName: string } const getInitialState = (): InitialState => { @@ -20,7 +21,8 @@ const getInitialState = (): InitialState => { agentConnected: false, language: "en-US", voiceType: "female", - chatItems: [] + chatItems: [], + graphName: "va.openai.azure" } } @@ -82,6 +84,9 @@ export const globalSlice = createSlice({ setLanguage: (state, action: PayloadAction) => { state.language = action.payload }, + setGraphName: (state, action: PayloadAction) => { + state.graphName = action.payload + }, setVoiceType: (state, action: PayloadAction) => { state.voiceType = action.payload }, @@ -94,7 +99,7 @@ export const globalSlice = createSlice({ export const { reset, setOptions, setRoomConnected, setAgentConnected, setVoiceType, - addChatItem, setThemeColor, setLanguage } = + addChatItem, setThemeColor, setLanguage, setGraphName } = globalSlice.actions export default globalSlice.reducer diff --git a/playground/src/types/index.ts b/playground/src/types/index.ts index 1f695c39..099e7069 100644 --- a/playground/src/types/index.ts +++ b/playground/src/types/index.ts @@ -33,6 +33,10 @@ export interface ITextItem { isFinal: boolean } +export interface GraphOptionItem { + label: string + value: string +} export interface LanguageOptionItem { label: string From 69960ca22471cfce55bd03d02d2a9d85d3b1281c Mon Sep 17 00:00:00 2001 From: sunshinexcode <24xinhui@163.com> Date: Wed, 14 Aug 2024 14:28:17 +0800 Subject: [PATCH 24/34] fix(): collection must start with a letter (#205) --- server/internal/http_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/internal/http_server.go b/server/internal/http_server.go index 75d6fb8c..aa6e1c31 100644 --- a/server/internal/http_server.go +++ b/server/internal/http_server.go @@ -323,7 +323,7 @@ func (s *HttpServer) handlerVectorDocumentUpload(c *gin.Context) { } // Generate collection - collection := fmt.Sprintf("%s_%d", gmd5.MustEncryptString(req.ChannelName), time.Now().UnixNano()) + collection := fmt.Sprintf("a%s_%d", gmd5.MustEncryptString(req.ChannelName), time.Now().UnixNano()) fileName := filepath.Base(file.Filename) // update worker From f78c76457b8cdbf07de884dd5b49fe00bb7ace34 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Tue, 13 Aug 2024 23:46:23 -0700 Subject: [PATCH 25/34] feat: calculate latency --- .../vector_storage_extension.py | 10 ++++++++++ .../extension/qwen_llm_python/qwen_llm_extension.py | 13 ++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py index 5cb53fe2..c89174a5 100644 --- a/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py +++ b/agents/addon/extension/aliyun_analyticdb_vector_storage/vector_storage_extension.py @@ -191,6 +191,7 @@ async def async_upsert_vector(self, rte: RteEnv, cmd: Cmd): rte.return_result(CmdResult.create(StatusCode.ERROR), cmd) async def async_query_vector(self, rte: RteEnv, cmd: Cmd): + start_time = datetime.now() m = Model(self.region_id, self.dbinstance_id, self.client) collection = cmd.get_property_string("collection_name") embedding = cmd.get_property_to_json("embedding") @@ -199,6 +200,15 @@ async def async_query_vector(self, rte: RteEnv, cmd: Cmd): response, error = await m.query_collection_data_async( collection, self.namespace, self.namespace_password, vector, top_k=top_k ) + logger.info( + "query_vector finished for collection {}, embedding len {}, err {}, cost {}ms".format( + collection, + len(embedding), + error, + int((datetime.now() - start_time).total_seconds() * 1000), + ) + ) + if error: return rte.return_result(CmdResult.create(StatusCode.ERROR), cmd) else: diff --git a/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py b/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py index bdc2e0cc..7a3860cb 100644 --- a/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py +++ b/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py @@ -88,7 +88,19 @@ def call_chat(self, rte: RteEnv, ts: datetime.time, cmd: Cmd): The incoming 'messages' will contains all the system prompt, chat history and question. """ + start_time = datetime.now() + curr_ttfs = None # time to first sentence + def callback(text: str, end_of_segment: bool): + nonlocal curr_ttfs + if curr_ttfs is None: + curr_ttfs = datetime.now() - start_time + logger.info( + "TTFS {}ms, sentence {} end_of_segment {}".format( + int(curr_ttfs.total_seconds() * 1000), text, end_of_segment + ) + ) + cmd_result = CmdResult.create(StatusCode.OK) cmd_result.set_property_string("text", text) if end_of_segment: @@ -100,7 +112,6 @@ def callback(text: str, end_of_segment: bool): messages_str = cmd.get_property_string("messages") messages = json.loads(messages_str) - # messages = [{"role": "user", "content": messages_str}] stream = False try: stream = cmd.get_property_bool("stream") From cbfac5895f28a3aaa7adec783429da6265d5eebd Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Tue, 13 Aug 2024 23:46:57 -0700 Subject: [PATCH 26/34] fix: cozy tts too slow --- agents/addon/extension/cosy_tts/cosy_tts_extension.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/agents/addon/extension/cosy_tts/cosy_tts_extension.py b/agents/addon/extension/cosy_tts/cosy_tts_extension.py index 07c15bf4..6d6d294d 100644 --- a/agents/addon/extension/cosy_tts/cosy_tts_extension.py +++ b/agents/addon/extension/cosy_tts/cosy_tts_extension.py @@ -31,7 +31,8 @@ def __init__(self, rte: RteEnv, sample_rate: int, need_interrupt_callback): self.rte = rte self.sample_rate = sample_rate self.frame_size = int(self.sample_rate * 1 * 2 / 100) - self.ts = datetime.now() + self.ts = datetime.now() # current task ts + self.init_ts = datetime.now() self.ttfb = None # time to first byte self.need_interrupt_callback = need_interrupt_callback self.closed = False @@ -41,7 +42,6 @@ def need_interrupt(self) -> bool: def set_input_ts(self, ts: datetime): self.ts = ts - self.ttfb = None # clear to calculate again def on_open(self): logger.info("websocket is open.") @@ -78,7 +78,7 @@ def on_data(self, data: bytes) -> None: if self.need_interrupt(): return if self.ttfb is None: - self.ttfb = datetime.now() - self.ts + self.ttfb = datetime.now() - self.init_ts logger.info("TTS TTFB {}ms".format(int(self.ttfb.total_seconds() * 1000))) # logger.info("audio result length: %d, %d", len(data), self.frame_size) @@ -207,8 +207,8 @@ def async_handle(self, rte: RteEnv): # last segment may have empty text but is_end is true tts.streaming_call(input_text) - # complete the streaming call to drain remained audio if end_of_segment is true - if end_of_segment: + # complete the streaming call to drain remained audio + if True: # end_of_segment: try: tts.streaming_complete() except Exception as e: From 4fc6e21b3624641f7f034abbe77870e68b2994c9 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Tue, 13 Aug 2024 23:48:54 -0700 Subject: [PATCH 27/34] feat: improve prompt --- .../llama_index_chat_engine/extension.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/agents/addon/extension/llama_index_chat_engine/extension.py b/agents/addon/extension/llama_index_chat_engine/extension.py index 718d6bb9..ce5578e6 100644 --- a/agents/addon/extension/llama_index_chat_engine/extension.py +++ b/agents/addon/extension/llama_index_chat_engine/extension.py @@ -193,7 +193,13 @@ def async_handle(self, rte: RteEnv): retriever=ASTRARetriever(rte=rte, coll=self.collection_name), memory=self.chat_memory, system_prompt=( - "You are an expert Q&A system that is trusted around the world.\n" + # "You are an expert Q&A system that is trusted around the world.\n" + "You are a voice assistant who talks in a conversational way and can chat with me like my friends. \n" + "I will speak to you in English or Chinese, and you will answer in the corrected and improved version of my text with the language I use. \n" + "Don’t talk like a robot, instead I would like you to talk like a real human with emotions. \n" + "I will use your answer for text-to-speech, so don’t return me any meaningless characters. \n" + "I want you to be helpful, when I’m asking you for advice, give me precise, practical and useful advice instead of being vague. \n" + "When giving me a list of options, express the options in a narrative way instead of bullet points.\n" "Always answer the query using the provided context information, " "and not prior knowledge.\n" "Some rules to follow:\n" @@ -206,7 +212,14 @@ def async_handle(self, rte: RteEnv): else: chat_engine = SimpleChatEngine.from_defaults( llm=ASTRALLM(rte=rte), - system_prompt="You are an expert Q&A system that is trusted around the world.\n", + system_prompt=( + "You are a voice assistant who talks in a conversational way and can chat with me like my friends. \n" + "I will speak to you in English or Chinese, and you will answer in the corrected and improved version of my text with the language I use. \n" + "Don’t talk like a robot, instead I would like you to talk like a real human with emotions. \n" + "I will use your answer for text-to-speech, so don’t return me any meaningless characters. \n" + "I want you to be helpful, when I’m asking you for advice, give me precise, practical and useful advice instead of being vague. \n" + "When giving me a list of options, express the options in a narrative way instead of bullet points.\n" + ), memory=self.chat_memory, ) From fd592e51a347128c6479a10018c26dde18861a22 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Wed, 14 Aug 2024 00:35:26 -0700 Subject: [PATCH 28/34] feat: add api --- .../http_server_python/manifest.json | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/agents/addon/extension/http_server_python/manifest.json b/agents/addon/extension/http_server_python/manifest.json index bfc16037..358d6e39 100644 --- a/agents/addon/extension/http_server_python/manifest.json +++ b/agents/addon/extension/http_server_python/manifest.json @@ -25,6 +25,40 @@ "listen_port": { "type": "int32" } - } + }, + "cmd_out": [ + { + "name": "update_querying_collection", + "property": { + "filename": { + "type": "string" + }, + "collection": { + "type": "string" + } + }, + "required": [ + "filename", + "collection" + ] + }, + { + "name": "file_chunk", + "property": { + "filename": { + "type": "string" + }, + "path": { + "type": "string" + }, + "collection": { + "type": "string" + } + }, + "required": [ + "path" + ] + } + ] } } \ No newline at end of file From 3b7fc6a421aa325961b8b1754d204994c79e89d3 Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Wed, 14 Aug 2024 16:09:45 +0800 Subject: [PATCH 29/34] fix: tmp move litellm to bak --- agents/addon/{extension => bak}/litellm_python/__init__.py | 0 agents/addon/{extension => bak}/litellm_python/extension.py | 0 agents/addon/{extension => bak}/litellm_python/litellm.py | 0 agents/addon/{extension => bak}/litellm_python/litellm_addon.py | 0 .../addon/{extension => bak}/litellm_python/litellm_extension.py | 0 agents/addon/{extension => bak}/litellm_python/log.py | 0 agents/addon/{extension => bak}/litellm_python/manifest.json | 0 agents/addon/{extension => bak}/litellm_python/requirements.txt | 0 agents/addon/{extension => bak}/litellm_python/utils.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename agents/addon/{extension => bak}/litellm_python/__init__.py (100%) rename agents/addon/{extension => bak}/litellm_python/extension.py (100%) rename agents/addon/{extension => bak}/litellm_python/litellm.py (100%) rename agents/addon/{extension => bak}/litellm_python/litellm_addon.py (100%) rename agents/addon/{extension => bak}/litellm_python/litellm_extension.py (100%) rename agents/addon/{extension => bak}/litellm_python/log.py (100%) rename agents/addon/{extension => bak}/litellm_python/manifest.json (100%) rename agents/addon/{extension => bak}/litellm_python/requirements.txt (100%) rename agents/addon/{extension => bak}/litellm_python/utils.py (100%) diff --git a/agents/addon/extension/litellm_python/__init__.py b/agents/addon/bak/litellm_python/__init__.py similarity index 100% rename from agents/addon/extension/litellm_python/__init__.py rename to agents/addon/bak/litellm_python/__init__.py diff --git a/agents/addon/extension/litellm_python/extension.py b/agents/addon/bak/litellm_python/extension.py similarity index 100% rename from agents/addon/extension/litellm_python/extension.py rename to agents/addon/bak/litellm_python/extension.py diff --git a/agents/addon/extension/litellm_python/litellm.py b/agents/addon/bak/litellm_python/litellm.py similarity index 100% rename from agents/addon/extension/litellm_python/litellm.py rename to agents/addon/bak/litellm_python/litellm.py diff --git a/agents/addon/extension/litellm_python/litellm_addon.py b/agents/addon/bak/litellm_python/litellm_addon.py similarity index 100% rename from agents/addon/extension/litellm_python/litellm_addon.py rename to agents/addon/bak/litellm_python/litellm_addon.py diff --git a/agents/addon/extension/litellm_python/litellm_extension.py b/agents/addon/bak/litellm_python/litellm_extension.py similarity index 100% rename from agents/addon/extension/litellm_python/litellm_extension.py rename to agents/addon/bak/litellm_python/litellm_extension.py diff --git a/agents/addon/extension/litellm_python/log.py b/agents/addon/bak/litellm_python/log.py similarity index 100% rename from agents/addon/extension/litellm_python/log.py rename to agents/addon/bak/litellm_python/log.py diff --git a/agents/addon/extension/litellm_python/manifest.json b/agents/addon/bak/litellm_python/manifest.json similarity index 100% rename from agents/addon/extension/litellm_python/manifest.json rename to agents/addon/bak/litellm_python/manifest.json diff --git a/agents/addon/extension/litellm_python/requirements.txt b/agents/addon/bak/litellm_python/requirements.txt similarity index 100% rename from agents/addon/extension/litellm_python/requirements.txt rename to agents/addon/bak/litellm_python/requirements.txt diff --git a/agents/addon/extension/litellm_python/utils.py b/agents/addon/bak/litellm_python/utils.py similarity index 100% rename from agents/addon/extension/litellm_python/utils.py rename to agents/addon/bak/litellm_python/utils.py From eacd25e2be5a06e8f31ed4cb3695c5a140117de6 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Wed, 14 Aug 2024 02:32:52 -0700 Subject: [PATCH 30/34] feat: interrput --- .../interrupt_detector_extension.py | 34 +++++++++----- .../llama_index_chat_engine/extension.py | 45 ++++++++++++++----- .../qwen_llm_python/qwen_llm_extension.py | 19 +++++--- agents/property.json.example | 45 +++++++++++++++---- 4 files changed, 107 insertions(+), 36 deletions(-) diff --git a/agents/addon/extension/interrupt_detector_python/interrupt_detector_extension.py b/agents/addon/extension/interrupt_detector_python/interrupt_detector_extension.py index 315aae52..f136891a 100644 --- a/agents/addon/extension/interrupt_detector_python/interrupt_detector_extension.py +++ b/agents/addon/extension/interrupt_detector_python/interrupt_detector_extension.py @@ -33,13 +33,31 @@ def on_stop(self, rte: RteEnv) -> None: logger.info("on_stop") rte.on_stop_done() + def send_flush_cmd(self, rte: RteEnv) -> None: + flush_cmd = Cmd.create(CMD_NAME_FLUSH) + rte.send_cmd( + flush_cmd, + lambda rte, result: logger.info("send_cmd done"), + ) + + logger.info(f"sent cmd: {CMD_NAME_FLUSH}") + def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: - logger.info("on_cmd") + cmd_name = cmd.get_name() + logger.info("on_cmd name {}".format(cmd_name)) + + # flush whatever cmd incoming at the moment + self.send_flush_cmd(rte) + + # then forward the cmd to downstream cmd_json = cmd.to_json() - logger.info("on_cmd json: " % cmd_json) + new_cmd = Cmd.create_from_json(cmd_json) + rte.send_cmd( + new_cmd, + lambda rte, result: logger.info("send_cmd done"), + ) cmd_result = CmdResult.create(StatusCode.OK) - cmd_result.set_property_string("detail", "success") rte.return_result(cmd_result, cmd) def on_data(self, rte: RteEnv, data: Data) -> None: @@ -73,15 +91,7 @@ def on_data(self, rte: RteEnv, data: Data) -> None: ) if final or len(text) >= 2: - flush_cmd = Cmd.create(CMD_NAME_FLUSH) - rte.send_cmd( - flush_cmd, - lambda rte, result: print( - "InterruptDetectorExtensionAddon send_cmd done" - ), - ) - - logger.info(f"sent cmd: {CMD_NAME_FLUSH}") + self.send_flush_cmd(rte) d = Data.create("text_data") d.set_property_bool(TEXT_DATA_FINAL_FIELD, final) diff --git a/agents/addon/extension/llama_index_chat_engine/extension.py b/agents/addon/extension/llama_index_chat_engine/extension.py index ce5578e6..4b84221b 100644 --- a/agents/addon/extension/llama_index_chat_engine/extension.py +++ b/agents/addon/extension/llama_index_chat_engine/extension.py @@ -25,6 +25,9 @@ PROPERTY_CHAT_MEMORY_TOKEN_LIMIT = "chat_memory_token_limit" PROPERTY_GREETING = "greeting" +TASK_TYPE_CHAT_REQUEST = "chat_request" +TASK_TYPE_GREETING = "greeting" + class LlamaIndexExtension(Extension): def __init__(self, name: str): @@ -33,8 +36,10 @@ def __init__(self, name: str): self.thread = None self.stop = False - self.collection_name = "" self.outdate_ts = datetime.now() + self.outdate_ts_lock = threading.Lock() + + self.collection_name = "" self.chat_memory_token_limit = 3000 self.chat_memory = None @@ -122,13 +127,15 @@ def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: # notify user file_chunked_text = "Your document has been processed. You can now start asking questions about your document. " - self._send_text_data(rte, file_chunked_text, True) + # self._send_text_data(rte, file_chunked_text, True) + self.queue.put((file_chunked_text, datetime.now(), TASK_TYPE_GREETING)) elif cmd_name == "file_chunk": self.collection_name = "" # clear current collection # notify user file_chunk_text = "Your document has been received. Please wait a moment while we process it for you. " - self._send_text_data(rte, file_chunk_text, True) + # self._send_text_data(rte, file_chunk_text, True) + self.queue.put((file_chunk_text, datetime.now(), TASK_TYPE_GREETING)) elif cmd_name == "update_querying_collection": coll = cmd.get_property_string("collection") logger.info( @@ -144,7 +151,10 @@ def on_cmd(self, rte: RteEnv, cmd: Cmd) -> None: update_querying_collection_text += ( "You can now start asking questions about your document. " ) - self._send_text_data(rte, update_querying_collection_text, True) + # self._send_text_data(rte, update_querying_collection_text, True) + self.queue.put( + (update_querying_collection_text, datetime.now(), TASK_TYPE_GREETING) + ) elif cmd_name == "flush": self.flush() @@ -168,7 +178,7 @@ def on_data(self, rte: RteEnv, data: Data) -> None: ts = datetime.now() logger.info("on_data text [%s], ts [%s]", inputText, ts) - self.queue.put((inputText, ts)) + self.queue.put((inputText, ts, TASK_TYPE_CHAT_REQUEST)) def async_handle(self, rte: RteEnv): logger.info("async_handle started") @@ -177,12 +187,21 @@ def async_handle(self, rte: RteEnv): value = self.queue.get() if value is None: break - input_text, ts = value - if ts < self.outdate_ts: + input_text, ts, task_type = value + + if ts < self.get_outdated_ts(): logger.info( - "text [%s] ts [%s] dropped due to outdated", input_text, ts + "text [{}] ts [{}] task_type [{}] dropped due to outdated".format( + input_text, ts, task_type + ) ) continue + + if task_type == TASK_TYPE_GREETING: + # send greeting text directly + self._send_text_data(rte, input_text, True) + continue + logger.info("process input text [%s] ts [%s]", input_text, ts) # prepare chat engine @@ -227,7 +246,7 @@ def async_handle(self, rte: RteEnv): for cur_token in resp.response_gen: if self.stop: break - if ts < self.outdate_ts: + if ts < self.get_outdated_ts(): logger.info( "stream_chat coming responses dropped due to outdated for input text [%s] ts [%s] ", input_text, @@ -246,6 +265,12 @@ def async_handle(self, rte: RteEnv): logger.info("async_handle stoped") def flush(self): - self.outdate_ts = datetime.now() + with self.outdate_ts_lock: + self.outdate_ts = datetime.now() + while not self.queue.empty(): self.queue.get() + + def get_outdated_ts(self): + with self.outdate_ts_lock: + return self.outdate_ts diff --git a/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py b/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py index 7a3860cb..6cd58f20 100644 --- a/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py +++ b/agents/addon/extension/qwen_llm_python/qwen_llm_extension.py @@ -34,9 +34,11 @@ def __init__(self, name: str): self.max_history = 10 self.stopped = False self.thread = None - self.outdate_ts = datetime.now() self.sentence_expr = re.compile(r".+?[,，.。!！?？:：]", re.DOTALL) + self.outdate_ts = datetime.now() + self.outdate_ts_lock = threading.Lock() + self.queue = queue.Queue() self.mutex = threading.Lock() @@ -62,7 +64,12 @@ def get_messages(self) -> List[Any]: return messages def need_interrupt(self, ts: datetime.time) -> bool: - return self.outdate_ts > ts and (self.outdate_ts - ts).total_seconds() > 1 + with self.outdate_ts_lock: + return self.outdate_ts > ts + + def get_outdate_ts(self) -> datetime: + with self.outdate_ts_lock: + return self.outdate_ts def complete_with_history(self, rte: RteEnv, ts: datetime.time, input_text: str): """ @@ -128,7 +135,7 @@ def stream_chat(self, ts: datetime.time, messages: List[Any], callback): logger.info("before stream_chat call {} {}".format(messages, ts)) if self.need_interrupt(ts): - logger.warning("out of date, %s, %s", self.outdate_ts, ts) + logger.warning("out of date, %s, %s", self.get_outdate_ts(), ts) return responses = dashscope.Generation.call( @@ -143,7 +150,7 @@ def stream_chat(self, ts: datetime.time, messages: List[Any], callback): partial = "" for response in responses: if self.need_interrupt(ts): - logger.warning("out of date, %s, %s", self.outdate_ts, ts) + logger.warning("out of date, %s, %s", self.get_outdate_ts(), ts) partial = "" # discard not sent break if response.status_code == HTTPStatus.OK: @@ -200,7 +207,9 @@ def on_stop(self, rte: RteEnv) -> None: rte.on_stop_done() def flush(self): - self.outdate_ts = datetime.now() + with self.outdate_ts_lock: + self.outdate_ts = datetime.now() + while not self.queue.empty(): self.queue.get() diff --git a/agents/property.json.example b/agents/property.json.example index 209d6a00..4c395374 100644 --- a/agents/property.json.example +++ b/agents/property.json.example @@ -1166,7 +1166,7 @@ } ] }, - { + { "name": "va.litellm.azure", "auto_start": true, "nodes": [ @@ -1537,6 +1537,37 @@ "extension": "llama_index" } ] + }, + { + "name": "file_chunk", + "dest": [ + { + "extension_group": "file_chunker", + "extension": "file_chunker" + }, + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + }, + { + "name": "file_chunked", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] + }, + { + "name": "update_querying_collection", + "dest": [ + { + "extension_group": "llama_index", + "extension": "llama_index" + } + ] } ], "data": [ @@ -1681,12 +1712,8 @@ "name": "file_chunk", "dest": [ { - "extension_group": "file_chunker", - "extension": "file_chunker" - }, - { - "extension_group": "llama_index", - "extension": "llama_index" + "extension_group": "interrupt_detector", + "extension": "interrupt_detector" } ] }, @@ -1694,8 +1721,8 @@ "name": "update_querying_collection", "dest": [ { - "extension_group": "llama_index", - "extension": "llama_index" + "extension_group": "interrupt_detector", + "extension": "interrupt_detector" } ] } From 34e8c6858e309a64bcbad4dc470509d2636fdf15 Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Wed, 14 Aug 2024 20:12:05 +0800 Subject: [PATCH 31/34] feat: pdf selection optimization --- playground/src/components/pdfSelect/index.tsx | 21 +++++++++++++++---- .../src/platform/pc/chat/index.module.scss | 2 +- playground/src/platform/pc/chat/index.tsx | 6 ++---- server/internal/worker.go | 4 ++-- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/playground/src/components/pdfSelect/index.tsx b/playground/src/components/pdfSelect/index.tsx index 31544e60..6564e493 100644 --- a/playground/src/components/pdfSelect/index.tsx +++ b/playground/src/components/pdfSelect/index.tsx @@ -13,21 +13,31 @@ const PdfSelect = () => { const options = useAppSelector(state => state.global.options) const { channel } = options const [pdfOptions, setPdfOptions] = useState([]) + const [selectedPdf, setSelectedPdf] = useState("") + const agentConnected = useAppSelector(state => state.global.agentConnected) useEffect(() => { - getPDFOptions() - }, []) + if(agentConnected) { + getPDFOptions() + } else { + setPdfOptions([]) + } + }, [agentConnected]) const getPDFOptions = async () => { const res = await apiGetDocumentList() - setPdfOptions(res.data.map((item: any) => { + setPdfOptions([{ + value: '', + label: 'Please select a PDF file' + }].concat(res.data.map((item: any) => { return { value: item.collection, label: item.file_name } - })) + }))) + setSelectedPdf('') } const onUploadSuccess = (data: IPdfData) => { @@ -35,6 +45,7 @@ const PdfSelect = () => { value: data.collection, label: data.fileName }]) + setSelectedPdf(data.collection) } const pdfDropdownRender = (menu: ReactElement) => { @@ -53,6 +64,7 @@ const PdfSelect = () => { if (!item) { return message.error("Please select a PDF file") } + setSelectedPdf(val) await apiUpdateDocument({ collection: val, fileName: item.label, @@ -64,6 +76,7 @@ const PdfSelect = () => { return } onChange={onSelectPdf} + value={selectedPdf} options={pdfOptions} dropdownRender={pdfDropdownRender} className={styles.pdfSelect} placeholder="Select a PDF file"> diff --git a/playground/src/platform/pc/chat/index.module.scss b/playground/src/platform/pc/chat/index.module.scss index ecc565d7..39c4956d 100644 --- a/playground/src/platform/pc/chat/index.module.scss +++ b/playground/src/platform/pc/chat/index.module.scss @@ -43,7 +43,7 @@ .right { display: flex; align-items: center; - gap: 5px; + gap: 10px; flex: 0 0 230px; justify-content: right; } diff --git a/playground/src/platform/pc/chat/index.tsx b/playground/src/platform/pc/chat/index.tsx index 1e218aa2..9a8238e8 100644 --- a/playground/src/platform/pc/chat/index.tsx +++ b/playground/src/platform/pc/chat/index.tsx @@ -42,16 +42,14 @@ const Chat = () => { return
- Agent Graph + + - Lang - -
diff --git a/server/internal/worker.go b/server/internal/worker.go index 583e5aa7..6e979e16 100644 --- a/server/internal/worker.go +++ b/server/internal/worker.go @@ -30,8 +30,8 @@ type Worker struct { type WorkerUpdateReq struct { RequestId string `form:"request_id,omitempty" json:"request_id,omitempty"` ChannelName string `form:"channel_name,omitempty" json:"channel_name,omitempty"` - Collection string `form:"collection,omitempty" json:"collection,omitempty"` - FileName string `form:"filename,omitempty" json:"filename,omitempty"` + Collection string `form:"collection,omitempty" json:"collection"` + FileName string `form:"filename,omitempty" json:"filename"` Path string `form:"path,omitempty" json:"path,omitempty"` Rte *WorkerUpdateReqRte `form:"rte,omitempty" json:"rte,omitempty"` } From 591b17976697c37aa2eb5fe9b1f9d0512215fa6b Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Wed, 14 Aug 2024 20:31:35 +0800 Subject: [PATCH 32/34] feat: replace cosy with azure --- agents/property.json.example | 17 ++++++++++++++--- docker-compose.yml | 8 +------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/agents/property.json.example b/agents/property.json.example index 4c395374..6eed13c4 100644 --- a/agents/property.json.example +++ b/agents/property.json.example @@ -1439,6 +1439,17 @@ "sample_rate": 16000 } }, + { + "type": "extension", + "extension_group": "tts", + "addon": "azure_tts", + "name": "azure_tts", + "property": { + "azure_subscription_key": "", + "azure_subscription_region": "", + "azure_synthesis_voice_name": "en-US-JaneNeural" + } + }, { "type": "extension", "extension_group": "chat_transcriber", @@ -1591,7 +1602,7 @@ "dest": [ { "extension_group": "tts", - "extension": "cosy_tts" + "extension": "azure_tts" }, { "extension_group": "chat_transcriber", @@ -1630,7 +1641,7 @@ }, { "extension_group": "tts", - "extension": "cosy_tts" + "extension": "azure_tts" } ] }, @@ -1665,7 +1676,7 @@ }, { "extension_group": "tts", - "extension": "cosy_tts", + "extension": "azure_tts", "pcm_frame": [ { "name": "pcm_frame", diff --git a/docker-compose.yml b/docker-compose.yml index 5c81a871..d001734f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,12 +16,6 @@ services: working_dir: /app env_file: - .env - astra_playground: - image: ghcr.io/rte-design/astra_playground:v0.2.0-15-g38e1738 - container_name: astra_playground - restart: always - ports: - - "3000:3000" astra_playground_dev: image: node:20-alpine @@ -29,7 +23,7 @@ services: restart: always command: sh -c "cd /app/playground && npm i && npm run dev" #build && npm run start" ports: - - "3002:3000" + - "3000:3000" volumes: - ./:/app astra_graph_designer: From 6586a3fa05ce09ec6e75beccc635138f66dc14f7 Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Wed, 14 Aug 2024 21:47:14 +0800 Subject: [PATCH 33/34] fix: fix pdf select --- playground/src/app/global.css | 3 ++- playground/src/components/pdfSelect/index.tsx | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/playground/src/app/global.css b/playground/src/app/global.css index daded34b..f7007287 100644 --- a/playground/src/app/global.css +++ b/playground/src/app/global.css @@ -31,7 +31,8 @@ a { } .ant-select-selector { - border: 1px solid #d9d9d9 !important; + border: 1px solid #272A2F !important; + background-color: #272A2F !important; } .ant-select-dropdown { diff --git a/playground/src/components/pdfSelect/index.tsx b/playground/src/components/pdfSelect/index.tsx index 6564e493..f593bf1d 100644 --- a/playground/src/components/pdfSelect/index.tsx +++ b/playground/src/components/pdfSelect/index.tsx @@ -13,7 +13,7 @@ const PdfSelect = () => { const options = useAppSelector(state => state.global.options) const { channel } = options const [pdfOptions, setPdfOptions] = useState([]) - const [selectedPdf, setSelectedPdf] = useState("") + const [selectedPdf, setSelectedPdf] = useState('') const agentConnected = useAppSelector(state => state.global.agentConnected) @@ -21,7 +21,10 @@ const PdfSelect = () => { if(agentConnected) { getPDFOptions() } else { - setPdfOptions([]) + setPdfOptions([{ + value: '', + label: 'Please select a PDF file' + }]) } }, [agentConnected]) From a7f73ef3492b4618d103eb897874f44eaa5b744b Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Wed, 14 Aug 2024 21:56:43 +0800 Subject: [PATCH 34/34] feat: add playground prebuilt image --- docker-compose.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d001734f..5c81a871 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,12 @@ services: working_dir: /app env_file: - .env + astra_playground: + image: ghcr.io/rte-design/astra_playground:v0.2.0-15-g38e1738 + container_name: astra_playground + restart: always + ports: + - "3000:3000" astra_playground_dev: image: node:20-alpine @@ -23,7 +29,7 @@ services: restart: always command: sh -c "cd /app/playground && npm i && npm run dev" #build && npm run start" ports: - - "3000:3000" + - "3002:3000" volumes: - ./:/app astra_graph_designer: