Merge remote-tracking branch 'origin' into dev/coze_demo

.devcontainer/devcontainer.json

@@ -2,7 +2,7 @@
 // README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
 {
     "name": "ten_agent_dev",
-    "image": "ghcr.io/ten-framework/ten_agent_build:0.2.3",
+    "image": "ghcr.io/ten-framework/ten_agent_build:0.2.4",
     "customizations": {
         "vscode": {
             "extensions": [

.env.example

@@ -100,7 +100,12 @@ MINIMAX_TTS_GROUP_ID=
 # OpenAI API key
 OPENAI_API_BASE=https://api.openai.com/v1
 OPENAI_API_KEY=
-OPENAI_MODEL=gpt-4o-mini
+OPENAI_MODEL=gpt-4o
+
+# set this to azure if you are using azure openai
+OPENAI_VENDOR=
+OPENAI_AZURE_ENDPOINT=
+OPENAI_AZURE_API_VERSION=
 
 # OpenAI API key for realtime API
 OPENAI_REALTIME_API_KEY=

.gitattributes

@@ -0,0 +1,5 @@
+# Treat all text files as text and normalize to LF in the repository
+* text=auto
+# Force LF for shell scripts or config files
+*.sh text eol=lf
+*.config text eol=lf

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/ten-framework/ten_agent_build:0.2.3 AS builder
+FROM ghcr.io/ten-framework/ten_agent_build:0.2.4 AS builder
 
 ARG SESSION_CONTROL_CONF=session_control.conf
 
@@ -7,7 +7,7 @@ WORKDIR /app
 COPY . .
 COPY agents/${SESSION_CONTROL_CONF} agents/session_control.conf
 
-RUN make clean && make build && \
+RUN task clean && task use && \
     cd agents && ./scripts/package.sh
 
 FROM ubuntu:22.04

README.md

@@ -36,17 +36,22 @@
 <br>
 <h2>🎉 TEN Agent with OpenAI Realtime API and RTC</h2>
 
-Try OpenAI Realtime API and RTC at [agent.theten.ai](https://agent.theten.ai).
+Try OpenAI Realtime API, **Weather Check** and **Web Search** at [agent.theten.ai](https://agent.theten.ai).
 
 Combining OpenAI Realtime API for ultra-low latency with RTC’s AI noise suppression ensures smooth, high-quality interactions. On top of that, the seamless integration of weather and news tools makes TEN Agent even more versatile. 
 
 ![TEN Agent with OpenAI Realtime API and RTC](https://github.com/TEN-framework/docs/blob/main/assets/gif/weather-and-news.gif?raw=true)
 
+<br>
+<h2>Usecases</h2>
+
+![Usecases](https://github.com/TEN-framework/docs/blob/main/assets/jpg/usecases.jpg?raw=true)
+
 <br>
 <h2>TEN Agent Features</h2>
 
-* **OpenAI Realtime API and RTC integration**:
-TEN Agent is the world-class multimodal AI agent to integrate the OpenAI Realtime API and RTC.
+* **OpenAI Realtime API and RTC Integration**:
+TEN Agent seamlessly integrates OpenAI's Realtime API with RTC capabilities, enabling natural, responsive conversations with ultra-low latency and high-quality audio processing.
 
 * **High-Performance Real-Time Multimodal Interactions**:
 Offers high-performance, low-latency solutions for complex audio-visual AI applications.
@@ -68,58 +73,43 @@ Manages and adjusts agent behavior in real-time for dynamic responsiveness.
 
 ![Ready-to-use Extensions](https://github.com/TEN-framework/docs/blob/main/assets/jpg/extensions.jpg?raw=true)
 
+
 <br>
-<h2>Stay Tuned</h2>
+<h2>How to build TEN Agent locally
 
-Before we get started, be sure to star our repository and get instant notifications for all new releases!
+### Prerequisites
 
-![TEN star us gif](https://github.com/TEN-framework/docs/blob/main/assets/gif/star_us_2.gif?raw=true)
+| Category | Requirements |
+|----------|-------------|
+| **Keys** | • Agora [ App ID ](https://docs.agora.io/en/video-calling/get-started/manage-agora-account?platform=web#create-an-agora-project) and [ App Certificate ](https://docs.agora.io/en/video-calling/get-started/manage-agora-account?platform=web#create-an-agora-project)(free minutes every month) <br>• [OpenAI](https://openai.com/index/openai-api/) API key<br>• [ Deepgram ](https://deepgram.com/) ASR (free credits available with signup)<br>• [ FishAudio ](https://fish.audio/) TTS (free credits available with signup)|
+| **Installation** | • [Docker](https://www.docker.com/) / [Docker Compose](https://docs.docker.com/compose/)<br>• [Node.js(LTS) v18](https://nodejs.org/en) |
+| **Minimum System Requirements** | • CPU >= 2 Core<br>• RAM >= 4 GB |
 
 <!-- <br>
-<h2>TEN Agent</h2>
 
-[TEN Agent](https://agent.theten.ai)
+### Windows settings(MUST READ)
 
-TEN Agent is a multimodal agent powered by [ TEN ](https://theten.ai), demonstrating its capabilities in speech, vision, and reasoning through  RAG from local documentation.
+On Windows, Git automatically adds a carriage return(\r) at the end of each line, which causes the `agents/bin/start: not found` error when running the server.
 
-And, if you choose to use OpenAI Realtime API, then the Vision and RAG won't be available.
+**If you encounter this issue**, follow these steps:
+1. Completely remove the current project folder
+2. Run the following command to disable Git's auto CRLF:
 
+```bash
+git config --global core.autocrlf false
+``` -->
 
-[![Showcase TEN multimodal agent](https://github.com/TEN-framework/docs/blob/main/assets/gif/features.gif?raw=true)](https://agent.theten.ai)
-<br> -->
+<!-- 3. Re-clone the project and start the process again -->
 
 <br>
-<h2>How to build TEN Agent locally
-
-### Prerequisites
-
-#### Keys
-- Agora [ App ID ](https://docs.agora.io/en/video-calling/get-started/manage-agora-account?platform=web#create-an-agora-project) and [ App Certificate ](https://docs.agora.io/en/video-calling/get-started/manage-agora-account?platform=web#create-an-agora-project)(certificate only required if enabled in the Agora Console)
-- [OpenAI](https://openai.com/index/openai-api/) API key
-- [ Deepgram ](https://deepgram.com/) ASR and [ FishAudio ](https://fish.audio/) TTS
-<!-- - Azure [SST(ASR)](https://azure.microsoft.com/en-us/products/ai-services/speech-to-text) and [TTS](https://azure.microsoft.com/en-us/products/ai-services/text-to-speech) API keys -->
-
-#### Installation
-  - [Docker](https://www.docker.com/) / [Docker Compose](https://docs.docker.com/compose/)
-  - [Node.js(LTS) v18](https://nodejs.org/en)
-
-#### Minimum system requirements
-  - CPU >= 2 Core
-  - RAM >= 4 GB
-
-#### MacOS: Docker setting on Apple Silicon
 
-You will need to uncheck "Use Rosetta for x86_64/amd64 emulation on Apple Silicon" option for Docker if you are on Apple Silicon. However, please note that build and connection times will be a little slower due to emulation when running on ARM systems. Once deployed to x64 (e.g. your Linux server) it will be much faster.
+### macOS: Docker setting on Apple Silicon
 
+For Apple Silicon Macs, uncheck "Use Rosetta for x86/amd64 emulation" in Docker settings. Note: This may result in slower build times on ARM, but performance will be normal when deployed to x64 servers.
 
 ![Docker Setting](https://github.com/TEN-framework/docs/blob/main/assets/gif/docker_setting.gif?raw=true)
 
-#### Windows: Configuring Git to handle line endings
-To avoid problems in `make run-server` later, you can configure Git to properly handle line endings on Windows.([more here](https://docs.github.com/en/get-started/getting-started-with-git/configuring-git-to-handle-line-endings?platform=windows))
-
-```bash
-git config --global core.autocrlf true
-```
+<br>
 
 ### Next step
 
@@ -134,8 +124,8 @@ cp ./.env.example ./.env
 #### 2. Setup API keys
 Open the `.env` file and fill in the `keys`. We recommend using [ Deepgram ASR ](https://deepgram.com/) and [ FishAudio TTS ](https://fish.audio/) as they are free to sign up for and offer free credits. Of course, you can also use other services, see the list in [.env.example](https://github.com/TEN-framework/ten-agent/blob/main/.env.example).
 ```bash
-# Agora App ID 
-# Agora App Certificate(only required if enabled in the Agora Console)
+# Agora App ID and Agora App Certificate
+# These are enabled by default when you create an Agora project
 AGORA_APP_ID=
 AGORA_APP_CERTIFICATE=
 
@@ -161,58 +151,40 @@ docker compose up -d
 Open up a separate terminal window, enter the container and build the agent:
 ```bash
 docker exec -it ten_agent_dev bash
-
-make build
+task use
 ```
 
 #### 5. Start the server
-Once the build is done, `make run-server` on port `8080`:
+Once the build is done, `task run` on port `8080/49483`:
 ```bash
-make run-server
+task run
 ```
 
 ### Finish and verify
 
 #### TEN Agent
 Open up [localhost:3000]( http://localhost:3000 ) in browser to play the TEN Agent.
 
-#### TEN Graph Designer
-
-Open up another tab go to [localhost:3001]( http://localhost:3001 ), and use Graph Designer to create, connect and edit extensions on canvas.
-
-Once you save the graph, you can return to [localhost:3000]( http://localhost:3000 ) and select the corresponding graph to view the changes.
+<br>
+<h2>How components work together</h2>
 
-![TEN Graph Designer](https://github.com/TEN-framework/docs/blob/main/assets/gif/hello_world_python.gif?raw=true)
+![Components Diagram](https://github.com/TEN-framework/docs/blob/main/assets/jpg/diagram.jpg?raw=true)
 
-<!-- <br>
-<h2>TEN Agent Comparison</h2>
-
-<div align="center">
+<br>
+<h2>Stay Tuned</h2>
 
-| **Features**                             | **TEN Agent** | **Pipecat** | **LiveKit:KITT** | **Vapi.ai** | **DailyBots** | **Play.ai** |
-|:----------------------------------------:|:-------:|:--------:|:-------------:|:----------------:|:----------------:|:----------------:|
-| **Vision**                               |   ✅    |    ❌    |      ❌       |     ❌     |     ❌      |     ❌       |
-| **Rich TTS Support for different languages** |   ✅    |    ❌    |      ❌       |     ❌      |     ❌      |     ❌      |
-| **Go support for extension**              |   ✅    |    ❌    |      ❌       |     ❌     |      ❌     |     ❌      |
-| **C++ support for extension**             |   ✅    |    ❌    |      ❌       |     ❌     |      ❌     |     ❌      |
-| **RAG support**                          |   ✅    |    ❌    |      ❌       |     ❌     |      ❌     |     ❌      |
-| **Workflow builder for extension**        |   ✅    |    ❌    |      ❌       |     ✅      |     ❌     |     ❌      |
-| **Rich LLM Support**                      |   ✅    |    ✅    |      ✅       |     ✅     |     ✅     |    ✅      |
-| **Python support for extension**          |   ✅    |    ✅    |      ✅       |     ✅     |     ✅      |     ✅     |
-| **Open source**                          |   ✅    |    ✅    |      ✅       |     ❌     |     ❌      |     ❌      |
+Before we get started, be sure to star our repository and get instant notifications for all new releases!
 
-</div> -->
+![TEN star us gif](https://github.com/TEN-framework/docs/blob/main/assets/gif/star_us_2.gif?raw=true)
 
 <br>
 
-
-<br>
 <h2>Join Community</h2>
 
 - [Discord](https://discord.gg/VnPftUzAMJ): Ideal for sharing your applications and engaging with the community.
 - [GitHub Discussion](https://github.com/TEN-framework/ten-agent/discussions): Perfect for providing feedback and asking questions.
 - [GitHub Issues](https://github.com/TEN-framework/ten-agent/issues): Best for reporting bugs and proposing new features. Refer to our [contribution guidelines](./docs/code-of-conduct/contributing.md) for more details.
-- [X (formerly Twitter)](https://img.shields.io/twitter/follow/TenFramework?logo=X&color=%20%23f5f5f5): Great for sharing your agents and interacting with the community.
+- [X](https://img.shields.io/twitter/follow/TenFramework?logo=X&color=%20%23f5f5f5): Great for sharing your agents and interacting with the community.
 
 
  <br>

Taskfile.yml

@@ -0,0 +1,67 @@
+version: '3'
+
+tasks:
+  clean:
+    desc: clean up
+    cmds:
+      - task: clean-agents
+      - task: clean-server
+
+  build:
+    desc: build
+    cmds:
+      - task: build-agent
+      - task: build-server
+
+  use:
+    desc: use agent, default 'agents/examples/demo'
+    vars:
+      AGENT: '{{.AGENT| default "agents/examples/default"}}'
+    cmds:
+      - ln -sf {{.USER_WORKING_DIR}}/{{.AGENT}}/manifest.json ./agents/
+      - ln -sf {{.USER_WORKING_DIR}}/{{.AGENT}}/property.json ./agents/
+      - task: build
+
+  run-server:
+    desc: run backend http server
+    cmds:
+      - /app/server/bin/api
+
+  run-gd-server:
+    desc: run tman dev http server for graph designer
+    dir: ./agents
+    cmds:
+      - tman dev-server
+
+  run:
+    desc: run servers
+    deps:
+      - task: run-server
+      - task: run-gd-server
+
+  build-agent:
+    desc: build agent
+    dir: ./agents
+    internal: true
+    cmds:
+      - ./scripts/install_deps_and_build.sh linux x64 && mv bin/main bin/worker
+
+  build-server:
+    desc: build server
+    dir: ./server
+    cmds:
+      - go mod tidy && go mod download && go build -o bin/api main.go
+
+  clean-agents:
+    desc: clean up agents
+    dir: ./agents
+    internal: true
+    cmds:
+      - rm -rf manifest.json property.json manifest-lock.json bin/main bin/worker out .release ten_packages/system/ten_runtime* ten_packages/system/agora_rtc_sdk ten_packages/system/azure_speech_sdk ten_packages/system/nlohmann_json ten_packages/extension/agora_rtc ten_packages/extension/agora_rtm ten_packages/extension/agora_sess_ctrl ten_packages/extension/azure_tts ten_packages/extension/py_init_extension_cpp
+
+  clean-server:
+    desc: clean up server
+    dir: ./server
+    internal: true
+    cmds:
+      - rm -rf bin

agents/.gitignore

@@ -34,3 +34,6 @@ xdump_config
 .vscode
 *.pyc
 *.pyc.*
+/manifest.json
+/manifest-lock.json
+/property.json