Skip to content

Commit

Permalink
Feat add deepgram_asr_python/cartesia_tts extensions (#358)
Browse files Browse the repository at this point in the history 
* feat(): add deepgram_asr_python/cartesia_tts extensions

* chore(): modify to use async and keep the connection

* chore(): upgrade ten_runtime_python version to 0.3

* chore(): add graph va_deepgram_openai_cartesia

* fix(): remove type extension_group
  • Loading branch information
@sunshinexcode
sunshinexcode authored Oct 29, 2024
1 parent b120e05 commit e875cbd
Show file tree
Hide file tree
Showing 20 changed files with 1,052 additions and 2 deletions.
10 changes: 9 additions & 1 deletion .env.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ WORKERS_MAX=100
# Worker quit timeout in seconds
WORKER_QUIT_TIMEOUT_SECONDES=60

# Agora App ID
# Agora App ID
# Agora App Certificate(only required if enabled in the Agora Console)
AGORA_APP_ID=
AGORA_APP_CERTIFICATE=
Expand Down Expand Up @@ -55,10 +55,18 @@ AZURE_STT_REGION=
AZURE_TTS_KEY=
AZURE_TTS_REGION=

# Extension: cartesia_tts
# Cartesia TTS key
CARTESIA_API_KEY=

# Extension: cosy_tts
# Cosy TTS key
COSY_TTS_KEY=

# Extension: deepgram_asr_python
# Deepgram ASR key
DEEPGRAM_API_KEY=

# Extension: elevenlabs_tts
# ElevenLabs TTS key
ELEVENLABS_TTS_KEY=
Expand Down
250 changes: 249 additions & 1 deletion agents/property.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3226,7 +3226,255 @@
]
}
]
},
{
"name": "va_deepgram_openai_cartesia",
"auto_start": false,
"nodes": [
{
"type": "extension",
"extension_group": "default",
"addon": "agora_rtc",
"name": "agora_rtc",
"property": {
"app_id": "${env:AGORA_APP_ID}",
"token": "<agora_token>",
"channel": "ten_agent_test",
"stream_id": 1234,
"remote_stream_id": 123,
"subscribe_audio": true,
"publish_audio": true,
"publish_data": true,
"enable_agora_asr": false,
"agora_asr_vendor_name": "microsoft",
"agora_asr_language": "en-US",
"agora_asr_vendor_key": "${env:AZURE_STT_KEY}",
"agora_asr_vendor_region": "${env:AZURE_STT_REGION}",
"agora_asr_session_control_file_path": "session_control.conf"
}
},
{
"type": "extension",
"extension_group": "asr",
"addon": "deepgram_asr_python",
"name": "deepgram_asr",
"property": {
"api_key": "${env:DEEPGRAM_API_KEY}",
"language": "en-US",
"model": "nova-2",
"sample_rate": "16000"
}
},
{
"type": "extension",
"extension_group": "chatgpt",
"addon": "openai_chatgpt",
"name": "openai_chatgpt",
"property": {
"base_url": "",
"api_key": "${env:OPENAI_API_KEY}",
"frequency_penalty": 0.9,
"model": "gpt-4o-mini",
"max_tokens": 512,
"prompt": "",
"proxy_url": "${env:OPENAI_PROXY_URL}",
"greeting": "TEN Agent connected. How can I help you today?",
"max_memory_length": 10
}
},
{
"type": "extension",
"extension_group": "tts",
"addon": "cartesia_tts",
"name": "cartesia_tts",
"property": {
"api_key": "${env:CARTESIA_API_KEY}",
"cartesia_version": "2024-06-10",
"model_id": "sonic-english",
"voice_id": "f9836c6e-a0bd-460e-9d3c-f7299fa60f94",
"sample_rate": "16000"
}
},
{
"type": "extension",
"extension_group": "default",
"addon": "interrupt_detector_python",
"name": "interrupt_detector"
},
{
"type": "extension",
"extension_group": "transcriber",
"addon": "message_collector",
"name": "message_collector"
}
],
"connections": [
{
"extension_group": "default",
"extension": "agora_rtc",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "asr",
"extension": "deepgram_asr"
},
{
"extension_group": "transcriber",
"extension": "message_collector"
}
]
}
],
"cmd": [
{
"name": "on_user_joined",
"dest": [
{
"extension_group": "asr",
"extension": "deepgram_asr"
}
]
},
{
"name": "on_user_left",
"dest": [
{
"extension_group": "asr",
"extension": "deepgram_asr"
}
]
},
{
"name": "on_connection_failure",
"dest": [
{
"extension_group": "asr",
"extension": "deepgram_asr"
}
]
}
]
},
{
"extension_group": "asr",
"extension": "deepgram_asr",
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "default",
"extension": "interrupt_detector"
},
{
"extension_group": "transcriber",
"extension": "message_collector"
}
]
}
]
},
{
"extension_group": "chatgpt",
"extension": "openai_chatgpt",
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "tts",
"extension": "cartesia_tts"
},
{
"extension_group": "transcriber",
"extension": "message_collector"
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "tts",
"extension": "cartesia_tts"
}
]
}
]
},
{
"extension_group": "transcriber",
"extension": "message_collector",
"data": [
{
"name": "data",
"dest": [
{
"extension_group": "default",
"extension": "agora_rtc"
}
]
}
]
},
{
"extension_group": "tts",
"extension": "cartesia_tts",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "default",
"extension": "agora_rtc"
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "default",
"extension": "agora_rtc"
}
]
}
]
},
{
"extension_group": "default",
"extension": "interrupt_detector",
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "chatgpt",
"extension": "openai_chatgpt"
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "chatgpt",
"extension": "openai_chatgpt"
}
]
}
]
}
]
}
]
}
}
}
6 changes: 6 additions & 0 deletions agents/ten_packages/extension/cartesia_tts/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from . import cartesia_tts_addon
from .extension import EXTENSION_NAME
from .log import logger


logger.info(f"{EXTENSION_NAME} extension loaded")
24 changes: 24 additions & 0 deletions agents/ten_packages/extension/cartesia_tts/cartesia_tts_addon.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#
#
# Agora Real Time Engagement
# Created by XinHui Li in 2024-07.
# Copyright (c) 2024 Agora IO. All rights reserved.
#
#

from ten import (
Addon,
register_addon_as_extension,
TenEnv,
)
from .extension import EXTENSION_NAME
from .log import logger


@register_addon_as_extension(EXTENSION_NAME)
class CartesiaTTSExtensionAddon(Addon):
def on_create_instance(self, ten: TenEnv, addon_name: str, context) -> None:
logger.info("on_create_instance")
from .cartesia_tts_extension import CartesiaTTSExtension

ten.on_create_instance_done(CartesiaTTSExtension(addon_name), context)
Loading

0 comments on commit e875cbd

Please sign in to comment.