Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support different TTS providers based on language selected #72

Merged
merged 7 commits into from
Jul 9, 2024
Merged

support different TTS providers based on language selected #72

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
93a1028
feat: add max_memory_length configuration option for OpenAI ChatGPT e…
sunshinexcode Jul 8, 2024
f097864
chore: support different TTS providers based on language selected
sunshinexcode Jul 9, 2024
fbbf616
fix(): fix default value
sunshinexcode Jul 9, 2024
3b0253d
chore: update Dockerfile and Makefile for improved build process
sunshinexcode Jul 9, 2024
b01916b
fix(): remove GOPROXY
sunshinexcode Jul 9, 2024
6f6b707
chore: remove sensitive logs
sunshinexcode Jul 9, 2024
23d4d89
chore(): rename tts
sunshinexcode Jul 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ include/
interface/
lib/
agents/manifest.json
agents/manifest.elevenlabs.json
!agents/addon/manifest.json
node_modules/
/out/
Expand Down
34 changes: 13 additions & 21 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,35 +1,27 @@
FROM agoraio/astra_agents_build:0.1.0 AS builder
FROM agoraio/astra_agents_build:latest AS builder

ARG SESSION_CONTROL_CONF=session_control.conf

WORKDIR /app

COPY . .
COPY agents/manifest.json.example agents/manifest.json
COPY agents/manifest.json.elevenlabs.example agents/manifest.elevenlabs.json
COPY agents/${SESSION_CONTROL_CONF} agents/session_control.conf

RUN make build && \
RUN export GOPROXY=https://proxy.golang.com.cn && make build && \
cd agents && ./scripts/package.sh

FROM agoraio/astra_agents_run:0.1.0

ARG AGORA_APP_ID
ARG AGORA_APP_CERTIFICATE
ARG MANIFEST_JSON_FILE=./agents/manifest.json
ARG AZURE_STT_KEY
ARG AZURE_STT_REGION
ARG OPENAI_API_KEY
ARG AZURE_TTS_KEY
ARG AZURE_TTS_REGION

ENV AGORA_APP_ID=${AGORA_APP_ID}
ENV AGORA_APP_CERTIFICATE=${AGORA_APP_CERTIFICATE}
ENV MANIFEST_JSON_FILE=${MANIFEST_JSON_FILE}
ENV AZURE_STT_KEY=${AZURE_STT_KEY}
ENV AZURE_STT_REGION=${AZURE_STT_REGION}
ENV OPENAI_API_KEY=${OPENAI_API_KEY}
ENV AZURE_TTS_KEY=${AZURE_TTS_KEY}
ENV AZURE_TTS_REGION=${AZURE_TTS_REGION}
FROM ubuntu:22.04

RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends \
libasound2 \
libgstreamer1.0-dev \
libunwind-dev \
libc++1 \
libssl-dev \
ca-certificates \
&& apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*

WORKDIR /app

Expand Down
5 changes: 2 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
MANIFEST_JSON_FILE := ./agents/manifest.json
PROJECT_NAME := astra
PROJECT_VERSION ?= "0.1."$(shell date -u +'%Y%m%d%H')
REGISTRY ?= agoraio/
Expand All @@ -19,10 +18,10 @@ build-server:

docker-build-server:
@echo ">> docker build server"
docker build -t $(REGISTRY)$(PROJECT_NAME):$(PROJECT_VERSION) -f Dockerfile_build .
docker build -t $(REGISTRY)$(PROJECT_NAME):$(PROJECT_VERSION) --platform linux/amd64 -f Dockerfile .
@echo ">> done"

run-server:
@echo ">> run server"
server/bin/api --manifestJsonFile=$(MANIFEST_JSON_FILE)
server/bin/api
@echo ">> done"
2 changes: 1 addition & 1 deletion agents/addon/extension/elevenlabs_tts/elevenlabs_tts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func defaultElevenlabsTTSConfig() elevenlabsTTSConfig {
return elevenlabsTTSConfig{
ApiKey: "",
ModelId: "eleven_multilingual_v2",
OptimizeStreamingLatency: 0.0,
OptimizeStreamingLatency: 0,
RequestTimeoutSeconds: 30,
SimilarityBoost: 0.75,
SpeakerBoost: false,
Expand Down
3 changes: 3 additions & 0 deletions agents/addon/extension/openai_chatgpt/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@
},
"greeting": {
"type": "string"
},
"max_memory_length": {
"type": "int64"
}
},
"data_in": [
Expand Down
16 changes: 12 additions & 4 deletions agents/addon/extension/openai_chatgpt/openai_chatgpt_extension.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,13 @@ const (
propertyMaxTokens = "max_tokens" // Optional
propertyGreeting = "greeting" // Optional
propertyProxyUrl = "proxy_url" // Optional

maxMemoryLength = 20
propertyMaxMemoryLength = "max_memory_length" // Optional
)

var (
memory []openai.ChatCompletionMessage
memoryChan chan openai.ChatCompletionMessage
memory []openai.ChatCompletionMessage
memoryChan chan openai.ChatCompletionMessage
maxMemoryLength = 10

outdateTs atomic.Int64
wg sync.WaitGroup
Expand Down Expand Up @@ -160,6 +160,14 @@ func (p *openaiChatGPTExtension) OnStart(rte rtego.Rte) {
slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyGreeting, err), logTag)
}

if propMaxMemoryLength, err := rte.GetPropertyInt64(propertyMaxMemoryLength); err != nil {
slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyMaxMemoryLength, err), logTag)
} else {
if propMaxMemoryLength > 0 {
maxMemoryLength = int(propMaxMemoryLength)
}
}

// create openaiChatGPT instance
openaiChatgpt, err := newOpenaiChatGPT(openaiChatGPTConfig)
if err != nil {
Expand Down
265 changes: 265 additions & 0 deletions agents/manifest.json.elevenlabs.example
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,265 @@
{
"type": "app",
"name": "astra_agents",
"version": "0.1.0",
"language": "go",
"dependencies": [
{
"type": "extension_group",
"name": "default_extension_group",
"version": "0.1.0"
},
{
"type": "extension",
"name": "agora_rtc",
"version": "0.1.1"
},
{
"type": "extension",
"name": "chat_transcriber",
"version": "0.1.0"
},
{
"type": "extension",
"name": "elevenlabs_tts",
"version": "0.1.0"
},
{
"type": "extension",
"name": "interrupt_detector",
"version": "0.1.0"
},
{
"type": "extension",
"name": "openai_chatgpt",
"version": "0.1.0"
}
],
"predefined_graphs": [
{
"name": "astra_agents",
"auto_start": true,
"nodes": [
{
"type": "extension",
"extension_group": "default",
"addon": "agora_rtc",
"name": "agora_rtc",
"property": {
"app_id": "<agora_appid>",
"token": "<agora_token>",
"channel": "astra_agents_test",
"stream_id": 1234,
"remote_stream_id": 123,
"subscribe_audio": true,
"publish_audio": true,
"publish_data": true,
"enable_agora_asr": true,
"agora_asr_vendor_name": "microsoft",
"agora_asr_language": "en-US",
"agora_asr_vendor_key": "<azure_stt_key>",
"agora_asr_vendor_region": "<azure_stt_region>",
"agora_asr_session_control_file_path": "session_control.conf"
}
},
{
"type": "extension",
"extension_group": "default",
"addon": "interrupt_detector",
"name": "interrupt_detector"
},
{
"type": "extension",
"extension_group": "chatgpt",
"addon": "openai_chatgpt",
"name": "openai_chatgpt",
"property": {
"base_url": "",
"api_key": "<openai_api_key>",
"frequency_penalty": 0.9,
"model": "gpt-3.5-turbo",
"max_tokens": 512,
"prompt": "",
"proxy_url": "",
"greeting": "ASTRA agent connected. How can i help you today?",
"max_memory_length": 10
}
},
{
"type": "extension",
"extension_group": "tts",
"addon": "elevenlabs_tts",
"name": "elevenlabs_tts",
"property": {
"api_key": "<elevenlabs_tts_key>",
"model_id": "eleven_multilingual_v2",
"optimize_streaming_latency": 0,
"request_timeout_seconds": 30,
"similarity_boost": 0.75,
"speaker_boost": false,
"stability": 0.5,
"style": 0.0,
"voice_id": "pNInz6obpgDQGcFmaJgB"
}
},
{
"type": "extension",
"extension_group": "transcriber",
"addon": "chat_transcriber",
"name": "chat_transcriber"
},
{
"type": "extension_group",
"addon": "default_extension_group",
"name": "default"
},
{
"type": "extension_group",
"addon": "default_extension_group",
"name": "chatgpt"
},
{
"type": "extension_group",
"addon": "default_extension_group",
"name": "tts"
},
{
"type": "extension_group",
"addon": "default_extension_group",
"name": "transcriber"
}
],
"connections": [
{
"extension_group": "default",
"extension": "agora_rtc",
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "default",
"extension": "interrupt_detector"
},
{
"extension_group": "chatgpt",
"extension": "openai_chatgpt"
},
{
"extension_group": "transcriber",
"extension": "chat_transcriber"
}
]
}
]
},
{
"extension_group": "chatgpt",
"extension": "openai_chatgpt",
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "tts",
"extension": "elevenlabs_tts"
},
{
"extension_group": "transcriber",
"extension": "chat_transcriber",
"cmd_conversions": [
{
"cmd": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "is_final",
"type": "fixed_value",
"value": "bool(true)"
},
{
"path": "stream_id",
"type": "fixed_value",
"value": "uint32(999)"
}
]
}
}
]
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "tts",
"extension": "elevenlabs_tts"
}
]
}
]
},
{
"extension_group": "tts",
"extension": "elevenlabs_tts",
"pcm_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "default",
"extension": "agora_rtc"
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "default",
"extension": "agora_rtc"
}
]
}
]
},
{
"extension_group": "transcriber",
"extension": "chat_transcriber",
"data": [
{
"name": "data",
"dest": [
{
"extension_group": "default",
"extension": "agora_rtc"
}
]
}
]
},
{
"extension_group": "default",
"extension": "interrupt_detector",
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "chatgpt",
"extension": "openai_chatgpt"
}
]
}
]
}
]
}
]
}
Loading