Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/tools #308

Merged
merged 9 commits into from
Oct 6, 2024
Merged

Feat/tools #308

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
700cbd1
init tool
tomasliu-agora Oct 2, 2024
c8eda45
tools wip
tomasliu-agora Oct 5, 2024
a729e98
fix python exception
tomasliu-agora Oct 6, 2024
23dc06d
feat: add tool in v2v
Oct 6, 2024
e8d6240
init tool
tomasliu-agora Oct 2, 2024
1266b38
fix python exception
tomasliu-agora Oct 6, 2024
f486037
fix
tomasliu-agora Oct 6, 2024
1c20910
remove out of date file
tomasliu-agora Oct 6, 2024
59ee505
remove standalone graph
tomasliu-agora Oct 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion agents/manifest-lock.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@
"type": "system",
"name": "nlohmann_json",
"version": "3.11.2",
"hash": "72b15822c7ea9deef5e7ad96216ac55e93f11b00466dd1943afd5ee276e99d19"
"hash": "72b15822c7ea9deef5e7ad96216ac55e93f11b00466dd1943afd5ee276e99d19",
"supports": []
},
{
"type": "system",
Expand Down
180 changes: 180 additions & 0 deletions agents/property.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2201,6 +2201,15 @@
"extension_group": "transcriber",
"addon": "message_collector",
"name": "message_collector"
},
{
"type": "extension",
"extension_group": "tools",
"addon": "weatherapi_tool_python",
"name": "weatherapi_tool_python",
"property": {
"api_key": "${env:WEATHERAPI_API_KEY}"
}
}
],
"connections": [
Expand All @@ -2219,6 +2228,21 @@
}
]
},
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
"extension_group": "llm",
"extension": "openai_v2v_python",
Expand Down Expand Up @@ -2253,6 +2277,162 @@
"extension": "agora_rtc"
}
]
},
{
"name": "tool_call",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python"
}
]
}
]
},
{
"extension_group": "transcriber",
"extension": "message_collector",
"data": [
{
"name": "data",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
}
]
}
]
},
{
"name": "va.openai.v2v.tools",
tomasliu-agora marked this conversation as resolved.
Show resolved Hide resolved
"auto_start": false,
"nodes": [
{
"type": "extension",
"extension_group": "rtc",
"addon": "agora_rtc",
"name": "agora_rtc",
"property": {
"app_id": "${env:AGORA_APP_ID}",
"token": "",
"channel": "astra_agents_test",
"stream_id": 1234,
"remote_stream_id": 123,
"subscribe_audio": true,
"publish_audio": true,
"publish_data": true,
"subscribe_audio_sample_rate": 24000
}
},
{
"type": "extension",
"extension_group": "llm",
"addon": "openai_v2v_python",
"name": "openai_v2v_python",
"property": {
"api_key": "${env:OPENAI_REALTIME_API_KEY}",
"temperature": 0.9,
"model": "gpt-4o-realtime-preview",
"max_tokens": 2048,
"voice": "alloy",
"language": "en-US",
"server_vad": true,
"dump": true
}
},
{
"type": "extension",
"extension_group": "transcriber",
"addon": "message_collector",
"name": "message_collector"
},
{
"type": "extension",
"extension_group": "tools",
"addon": "weatherapi_tool_python",
"name": "weatherapi_tool_python",
"property": {
"api_key": "${env:WEATHERAPI_API_KEY}"
}
}
],
"connections": [
{
"extension_group": "rtc",
"extension": "agora_rtc",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
"extension_group": "llm",
"extension": "openai_v2v_python",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
}
],
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "transcriber",
"extension": "message_collector"
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
},
{
"name": "tool_call",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python"
}
]
}
]
},
Expand Down
175 changes: 175 additions & 0 deletions agents/ten_packages/extension/openai_v2v_python/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
import asyncio
import base64
import json
import os
from typing import Any, AsyncGenerator

import uuid
import aiohttp
from . import messages

from .log import logger

DEFAULT_MODEL = "gpt-4o-realtime-preview"

DEFAULT_INSTRUCTION = '''
You are a helpful voice assistant, user can see transcription through the chat box. Your name is TEN Agent. You are built by TEN Framework which is a powerful realtime multimodal agent framework. User's input will mainly be {language}, and your response must be {language}.
{tools}
'''

def smart_str(s: str, max_field_len: int = 128) -> str:
"""parse string as json, truncate data field to 128 characters, reserialize"""
try:
data = json.loads(s)
if "delta" in data:
key = "delta"
elif "audio" in data:
key = "audio"
else:
return s

if len(data[key]) > max_field_len:
data[key] = data[key][:max_field_len] + "..."
return json.dumps(data)
except json.JSONDecodeError:
return s


def generate_client_event_id() -> str:
return str(uuid.uuid4())

class RealtimeApiConfig:
def __init__(
self,
base_uri: str = "wss://api.openai.com",
api_key: str | None = None,
path: str = "/v1/realtime",
verbose: bool = False,
model: str=DEFAULT_MODEL,
language: str = "en-US",
system_message: str=DEFAULT_INSTRUCTION,
temperature: float =0.5,
max_tokens: int =1024,
voice: messages.Voices = messages.Voices.Alloy,
server_vad:bool=True,
):
self.base_uri = base_uri
self.api_key = api_key
self.path = path
self.verbose = verbose
self.model = model
self.language = language
self.system_message = system_message
self.temperature = temperature
self.max_tokens = max_tokens
self.voice = voice
self.server_vad = server_vad

def build_ctx(self) -> dict:
return {
"language": self.language
}

class RealtimeApiClient:
tomasliu-agora marked this conversation as resolved.
Show resolved Hide resolved
def __init__(
self,
base_uri: str,
api_key: str | None = None,
path: str = "/v1/realtime",
model: str = DEFAULT_MODEL,
verbose: bool = False,
session: aiohttp.ClientSession | None = None,
):
is_local = (
base_uri.startswith("localhost")
or base_uri.startswith("127.0.0.1")
or base_uri.startswith("0.0.0.0")
)
has_scheme = base_uri.startswith("ws://") or base_uri.startswith("wss://")
self.url = f"{base_uri}{path}"
if model:
self.url += f"?model={model}"
if verbose:
logger.info(f"URL: {self.url} {is_local=} {has_scheme=}")

if not has_scheme:
if is_local:
self.url = f"ws://{self.url}"
else:
self.url = f"wss://{self.url}"

self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
self.websocket: aiohttp.ClientWebSocketResponse | None = None
self.verbose = verbose
self.session = session or aiohttp.ClientSession()

async def __aenter__(self) -> "RealtimeApiClient":
await self.connect()
return self

async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> bool:
await self.shutdown()
return False

async def connect(self):
auth = aiohttp.BasicAuth("", self.api_key) if self.api_key else None

headers = {"OpenAI-Beta": "realtime=v1"}
if "PROD_COMPLETIONS_API_KEY" in os.environ:
headers["X-Prod-Completions-Api-Key"] = os.environ["PROD_COMPLETIONS_API_KEY"]
elif "OPENAI_API_KEY" in os.environ:
headers["X-Prod-Completions-Api-Key"] = os.environ["OPENAI_API_KEY"]
if "PROD_COMPLETIONS_ORG_ID" in os.environ:
headers["X-Prod-Completions-Org-Id"] = os.environ["PROD_COMPLETIONS_ORG_ID"]
if headers:
logger.debug("Using X-Prod-Completions-* headers for api credentials")

self.websocket = await self.session.ws_connect(
url=self.url,
auth=auth,
headers=headers,
)

async def send_audio_data(self, audio_data: bytes):
"""audio_data is assumed to be pcm16 24kHz mono little-endian"""
base64_audio_data = base64.b64encode(audio_data).decode("utf-8")
message = messages.InputAudioBufferAppend(audio=base64_audio_data)
await self.send_message(message)

async def send_message(self, message: messages.ClientToServerMessage):
assert self.websocket is not None
if message.event_id is None:
message.event_id = generate_client_event_id()
message_str = message.model_dump_json()
if self.verbose:
logger.info(f"-> {smart_str(message_str)}")
await self.websocket.send_str(message_str)

async def listen(self) -> AsyncGenerator[messages.RealtimeMessage, None]:
assert self.websocket is not None
if self.verbose:
logger.info("Listening for realtimeapi messages")
try:
async for msg in self.websocket:
if msg.type == aiohttp.WSMsgType.TEXT:
if self.verbose:
logger.info(f"<- {smart_str(msg.data)}")
yield self.handle_server_message(msg.data)
elif msg.type == aiohttp.WSMsgType.ERROR:
logger.error("Error during receive: %s", self.websocket.exception())
break
except asyncio.CancelledError:
logger.info("Receive messages task cancelled")

def handle_server_message(self, message: str) -> messages.ServerToClientMessage:
try:
return messages.parse_server_message(message)
except Exception as e:
logger.error("Error handling message: " + str(e))
#raise e

async def shutdown(self):
# Close the websocket connection if it exists
if self.websocket:
await self.websocket.close()
self.websocket = None
Loading