Enable python

.devcontainer/devcontainer.json

@@ -2,7 +2,7 @@
 // README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
 {
     "name": "astra",
-    "image": "agoraio/astra_agents_build",
+    "image": "ghcr.io/rte-design/astra_agents_build",
 
     "customizations": {
         "vscode": {

.github/workflows/build-docker.yaml

@@ -2,7 +2,7 @@ name: Build Docker
 
 on: 
   push:
-    branches: [ "main" ]
+    branches: [ "main", "python-experimental-develop" ]
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
     paths-ignore:
@@ -36,6 +36,6 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
         registry: ghcr.io
-        tags: "latest,${{ steps.pre-step.outputs.image-tag }}"
+        tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}"
         no_push: ${{ github.event_name == 'pull_request' }}
 

.gitignore

@@ -20,8 +20,11 @@ crash_context_v1
 include/
 interface/
 lib/
+lib64
 agents/manifest.json
 agents/manifest.elevenlabs.json
+agents/manifest.cn.json
+agents/manifest.en.json
 !agents/addon/manifest.json
 node_modules/
 /out/
@@ -32,3 +35,5 @@ xdump_config
 .vscode/
 speechsdk/
 SpeechSDK-Linux.tar.gz
+pyvenv.cfg
+xdump_config

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "python.analysis.extraPaths": [
+      "./agents/interface",
+    ],
+    "editor.formatOnSave": true,
+  }

Dockerfile

@@ -1,12 +1,14 @@
-FROM agoraio/astra_agents_build:latest AS builder
+FROM ghcr.io/rte-design/astra_agents_build:0.3.3 AS builder
 
 ARG SESSION_CONTROL_CONF=session_control.conf
 
 WORKDIR /app
 
 COPY . .
 COPY agents/manifest.json.example agents/manifest.json
-COPY agents/manifest.json.elevenlabs.example agents/manifest.elevenlabs.json
+# COPY agents/manifest.json.elevenlabs.example agents/manifest.elevenlabs.json
+COPY agents/manifest.json.cn.example agents/manifest.cn.json
+COPY agents/manifest.json.en.example agents/manifest.en.json
 COPY agents/${SESSION_CONTROL_CONF} agents/session_control.conf
 
 RUN make build && \
@@ -20,13 +22,20 @@ RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommend
     libunwind-dev \
     libc++1 \
     libssl-dev \
+    python3 \
+    python3-venv \
+    python3-pip \
+    python3-dev \
+    jq \
     ca-certificates \
     && apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
 
 WORKDIR /app
 
 COPY --from=builder /app/agents/.release/ agents/
 COPY --from=builder /app/server/bin/api /app/server/bin/api
+COPY --from=builder /usr/local/lib /usr/local/lib
+COPY --from=builder /usr/lib/python3 /usr/lib/python3
 
 EXPOSE 8080
 

Makefile

@@ -16,6 +16,13 @@ build-server:
 	cd server && go mod tidy && go mod download && go build -o bin/api main.go
 	@echo ">> done"
 
+clean: clean-agents
+
+clean-agents:
+	@echo ">> clean agents"
+	rm -rf agents/manifest.json agents/bin agents/out agents/interface agents/include agents/lib agents/lib64 agents/addon/system agents/addon/extension_group agents/.release
+	@echo ">> done"
+
 docker-build-server:
 	@echo ">> docker build server"
 	docker build -t $(REGISTRY)$(PROJECT_NAME):$(PROJECT_VERSION) --platform linux/amd64 -f Dockerfile .

README.md

@@ -137,9 +137,14 @@ You need to prepare the proper `manifest.json` file first.
 ```bash
 # Rename manifest example
 cp ./agents/manifest.json.example ./agents/manifest.json
+cp ./agents/manifest.json.en.example ./agents/manifest.en.json
+cp ./agents/manifest.json.cn.example ./agents/manifest.cn.json
 
-# Pull the docker image with dev tools and mount your current folder as workspace
-docker run -itd -v $(pwd):/app -w /app -p 8080:8080 --name astra_agents_dev agoraio/astra_agents_build
+# pull the docker image with dev tools and mount your current folder as workspace
+docker run -itd -v $(pwd):/app -w /app -p 8080:8080 --name astra_agents_dev ghcr.io/rte-design/astra_agents_build
+
+# for windows git bash
+# docker run -itd -v //$(pwd):/app -w //app -p 8080:8080 --name astra_agents_dev ghcr.io/rte-design/astra_agents_build
 
 # Enter docker image
 docker exec -it astra_agents_dev bash
@@ -155,19 +160,22 @@ The above code generates an agent executable. To customize your prompts and Open
 Once you have made the necessary changes, you can use the following commands to start a server. You can then test it out using the ASTRA voice agent from the showcase.
 
 ```bash
-
+# TODO: need to refactor the contents 
 # Agora App ID and Agora App Certificate
 export AGORA_APP_ID=<your_agora_appid>
 export AGORA_APP_CERTIFICATE=<your_agora_app_certificate>
 
 # OpenAI API key
 export OPENAI_API_KEY=<your_openai_api_key>
+# Or QWEN key
+export QWEN_API_KEY=<your_qwern_api_key>
 
 # Azure STT key and region
 export AZURE_STT_KEY=<your_azure_stt_key>
 export AZURE_STT_REGION=<your_azure_stt_region>
 
-# Here are two TTS options, either one will work
+# TTS
+# Here are three TTS options, either one will work
 # Make sure to comment out the one you don't use
 
 # 1. using Azure
@@ -179,13 +187,48 @@ export AZURE_TTS_REGION=<your_azure_tts_region>
 export TTS_VENDOR_ENGLISH=elevenlabs
 export ELEVENLABS_TTS_KEY=<your_elevanlabs_tts_key>
 
+# 3. using Cosy
+export COSY_TTS_KEY=<your_cosy_tts_key>
+
 # agent is ready to start on port 8080
 
 make run-server
 ```
 
 🎉 Congratulations! You have created your first personalized voice agent.
 
+<h3>Quick Agent Customize Test</h3>
+The default agent control is managed via server gateway. For quick testing, you can also run the agent directly.
+
+```
+
+# rename manifest example
+cp ./agents/manifest.json.example ./agents/manifest.json
+cp ./agents/manifest.json.en.example ./agents/manifest.en.json
+cp ./agents/manifest.json.cn.example ./agents/manifest.cn.json
+
+# pull the docker image with dev tools and mount your current folder as workspace
+docker run -itd -v $(pwd):/app -w /app -p 8080:8080 --name astra_agents_dev ghcr.io/rte-design/astra_agents_build
+
+# for windows git bash
+# docker run -itd -v //$(pwd):/app -w //app -p 8080:8080 --name astra_agents_dev ghcr.io/rte-design/astra_agents_build
+
+# enter docker image
+docker exec -it astra_agents_dev bash
+
+make build
+
+cd ./agents
+# manipulate values in manifest.json to replace <agora_appid>, <qwern_api_key>, <stt_api_key>, <stt_region> with your keys
+./bin/start
+```
+
+use [https://webdemo.agora.io/](https://webdemo.agora.io/) to quickly test.
+
+Note the `channel` and `remote_stream_id` needs to match with the one you use on `https://webdemo.agora.io/`
+
+<br>
+<h2>ASTRA Service</h2>
 <h3>Discover More</h3>
 
 Now that you’ve created your first AI agent, the creativity doesn’t stop here. To develop more amazing agents, you’ll need an advanced understanding of how the ASTRA works under the hood. Please refer to the [ ASTRA architecture documentation ](./docs/astra-architecture.md).

agents/.gitignore

@@ -1,7 +1,9 @@
 *.log
 addon/extension_group/
 addon/extension/agora_rtc
+addon/extension/py_init_extension_cpp
 addon/system
+.rte
 agoradns.dat
 agorareport.dat
 bin/
@@ -23,3 +25,4 @@ lib/
 session_control.conf.agora
 xdump_config
 .vscode
+*.pyc

agents/addon/extension/azure_tts/manifest.json

@@ -1,13 +1,13 @@
 {
     "type": "extension",
     "name": "azure_tts",
-    "version": "0.1.0",
+    "version": "0.2.0",
     "language": "cpp",
     "dependencies": [
         {
             "type": "system",
             "name": "rte_runtime",
-            "version": "0.1.0"
+            "version": "0.3.1"
         },
         {
             "type": "system",

agents/addon/extension/azure_tts/src/main.cc

@@ -13,9 +13,6 @@
 
 #include "log.h"
 #include "macro/check.h"
-#include "rte_runtime/binding/cpp/internal/msg/cmd/cmd.h"
-#include "rte_runtime/binding/cpp/internal/msg/pcm_frame.h"
-#include "rte_runtime/binding/cpp/internal/rte_proxy.h"
 #include "rte_runtime/binding/cpp/rte.h"
 #include "tts.h"
 
@@ -31,7 +28,7 @@ class azure_tts_extension_t : public rte::extension_t {
   //   - azure_subscription_key
   //   - azure_subscription_region
   //   - azure_synthesis_voice_name
-  void on_start(rte::rte_t &rte) override {
+  void on_start(rte::rte_env_t &rte) override {
     AZURE_TTS_LOGI("start");
 
     // read properties
@@ -45,7 +42,7 @@ class azure_tts_extension_t : public rte::extension_t {
       return;
     }
 
-    rte_proxy_ = std::unique_ptr<rte::rte_proxy_t>(rte::rte_proxy_t::create(rte));
+    rte_proxy_ = std::unique_ptr<rte::rte_env_proxy_t>(rte::rte_env_proxy_t::create(rte));
     RTE_ASSERT(rte_proxy_ != nullptr, "rte_proxy should not be nullptr");
 
     // pcm parameters
@@ -69,17 +66,17 @@ class azure_tts_extension_t : public rte::extension_t {
           pcm_frame->set_data_fmt(RTE_PCM_FRAME_DATA_FMT_INTERLEAVE);
           pcm_frame->set_samples_per_channel(samples_per_10ms);
           pcm_frame->alloc_buf(pcm_frame_size);
-          rte::buf_t borrowed_buf = pcm_frame->borrow_buf(0);
+          rte::buf_t borrowed_buf = pcm_frame->lock_buf(0);
           auto *buf = borrowed_buf.data();
           if (buf != nullptr) {
             memset(buf, 0, pcm_frame_size);  // fill empty if size is not enough for 10ms
             memcpy(buf, data, size);
           }
-          pcm_frame->give_back_buf(borrowed_buf);
+          pcm_frame->unlock_buf(borrowed_buf);
 
           auto pcm_frame_shared = std::make_shared<std::unique_ptr<rte::pcm_frame_t>>(std::move(pcm_frame));
           rte_proxy->notify(
-              [frame = std::move(pcm_frame_shared)](rte::rte_t &rte) { rte.send_pcm_frame(std::move(*frame)); });
+              [frame = std::move(pcm_frame_shared)](rte::rte_env_t &rte) { rte.send_pcm_frame(std::move(*frame)); });
         };
 
 
@@ -104,9 +101,9 @@ class azure_tts_extension_t : public rte::extension_t {
   //  - name: flush
   //    example:
   //      {"name": "flush"}
-  void on_cmd(rte::rte_t &rte, std::unique_ptr<rte::cmd_t> cmd) override {
+  void on_cmd(rte::rte_env_t &rte, std::unique_ptr<rte::cmd_t> cmd) override {
 
-    std::string command = cmd->get_msg_name();
+    std::string command = cmd->get_name();
     AZURE_TTS_LOGI("%s", command.c_str());
 
     if (command == kCmdNameFlush) {
@@ -115,14 +112,14 @@ class azure_tts_extension_t : public rte::extension_t {
 
       // passthrough cmd
       auto ret = rte.send_cmd(rte::cmd_t::create(kCmdNameFlush.c_str()));
-      if (ret != RTE_STATUS_CODE_OK) {
-        AZURE_TTS_LOGE("Failed to send cmd %s, ret:%d", kCmdNameFlush.c_str(), int(ret));
-        rte.return_string(RTE_STATUS_CODE_ERROR, "Failed to send cmd", std::move(cmd));
+      if (!ret) {
+        AZURE_TTS_LOGE("Failed to send cmd %s", kCmdNameFlush.c_str());
+        rte.return_result(rte::cmd_result_t::create(RTE_STATUS_CODE_ERROR), std::move(cmd));
       } else {
-        rte.return_string(RTE_STATUS_CODE_OK, "ok", std::move(cmd));
+        rte.return_result(rte::cmd_result_t::create(RTE_STATUS_CODE_OK), std::move(cmd));
       }
     } else {
-      rte.return_string(RTE_STATUS_CODE_OK, "unregistered cmd", std::move(cmd));
+      rte.return_result(rte::cmd_result_t::create(RTE_STATUS_CODE_OK), std::move(cmd));
     }
   }
 
@@ -131,7 +128,7 @@ class azure_tts_extension_t : public rte::extension_t {
   //  - name: text_data
   //    example:
   //      {"name": "text_data", "properties": {"text": "hello"}
-  void on_data(rte::rte_t &rte, std::unique_ptr<rte::data_t> data) override {
+  void on_data(rte::rte_env_t &rte, std::unique_ptr<rte::data_t> data) override {
 
     auto text = data->get_property_string(kDataFieldText.c_str());
     if (text.empty()) {
@@ -145,27 +142,28 @@ class azure_tts_extension_t : public rte::extension_t {
   }
 
   // on_stop will be called when the extension is stopping.
-  void on_stop(rte::rte_t &rte) override {
+  void on_stop(rte::rte_env_t &rte) override {
     AZURE_TTS_LOGI("stop");
     if (azure_tts_) {
       azure_tts_->Stop();
       azure_tts_ = nullptr;
     }
+    rte_proxy_.reset();
 
     // Extension stop.
     rte.on_stop_done();
     AZURE_TTS_LOGI("stop done");
   }
 
  private:
-  std::unique_ptr<rte::rte_proxy_t> rte_proxy_;
+  std::unique_ptr<rte::rte_env_proxy_t> rte_proxy_;
 
   std::unique_ptr<AzureTTS> azure_tts_;
 
   const std::string kCmdNameFlush{"flush"};
   const std::string kDataFieldText{"text"};
 };
 
-RTE_CXX_REGISTER_ADDON_AS_EXTENSION(azure_tts, azure_tts_extension_t);
+RTE_CPP_REGISTER_ADDON_AS_EXTENSION(azure_tts, azure_tts_extension_t);
 
 }  // namespace azure_tts_extension

agents/addon/extension/azure_tts/src/tts.cc

@@ -36,18 +36,14 @@ bool AzureTTS::Start() {
 
       {
         std::unique_lock<std::mutex> lk(tasks_mutex_);
-        tasks_cv_.wait(lk, [this]() { return !tasks_.empty(); });
-        if (tasks_.empty()) {
-          continue;
+        tasks_cv_.wait(lk, [this]() { return stop_.load() || !tasks_.empty(); });
+        if (stop_.load()) {
+          break;
         }
         task = std::move(tasks_.front());
         tasks_.pop();
       }
 
-      if (stop_.load()) {
-        break;
-      }
-
       SpeechText(task->text, task->ts);
     }
 

agents/addon/extension/bedrock_llm_python/README.md

@@ -0,0 +1,12 @@
+## Amazon Bedrock LLM Extension
+
+### Configurations
+
+You can config this extension by providing following environments:
+
+| Env | Required | Default | Notes |
+| -- | -- | -- | -- |
+| AWS_REGION | No | us-east-1 | The Region of Amazon Bedrock service you want to use. |
+| AWS_ACCESS_KEY_ID | No | - | Access Key of your IAM User, make sure you've set proper permissions to [invoke Bedrock models](https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html) and gain [models access](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) in Bedrock. Will use default credentials provider if not provided. Check [document](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html).  |
+| AWS_SECRET_ACCESS_KEY | No | - | Secret Key of your IAM User, make sure you've set proper permissions to [invoke Bedrock models](https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html) and gain [models access](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) in Bedrock. Will use default credentials provider if not provided. Check [document](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html). |
+| AWS_BEDROCK_MODEL | No | Claude 3.5(anthropic.claude-3-5-sonnet-20240620-v1:0) | Bedrock model id, check [docuement](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html#model-ids-arns).  |

agents/addon/extension/bedrock_llm_python/__init__.py

@@ -0,0 +1,4 @@
+from . import bedrock_llm_extension
+from .log import logger
+
+logger.info("bedrock_llm_python extension loaded")