Skip to content

Commit

Permalink
update graph
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomasliu-agora
tomasliu-agora committed Oct 25, 2024
1 parent 5149dc6 commit 76e5ded
Show file tree
Hide file tree
Showing 5 changed files with 328 additions and 14 deletions.
4 changes: 4 additions & 0 deletions .env.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ ELEVENLABS_TTS_KEY=
# Gemini API key
GEMINI_API_KEY=

# Extension: azure_vision_python
# Azure vision key
AZURE_VISION_KEY=

# Extension: litellm
# Using Environment Variables, refer to https://docs.litellm.ai/docs/providers
# For example:
Expand Down
13 changes: 0 additions & 13 deletions agents/manifest-lock.json
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,6 @@
{
"version": 1,
"packages": [
{
"type": "extension",
"name": "default_extension_python",
"version": "0.2.0",
"hash": "e9a45737eb95cd9deca485f6d4274b46533a8346d59edf1e1b7c8cd29754a15b",
"dependencies": [
{
"type": "system",
"name": "ten_runtime_python"
}
],
"supports": []
},
{
"type": "system",
"name": "ten_runtime_go",
Expand Down
319 changes: 319 additions & 0 deletions agents/property.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2659,6 +2659,325 @@
]
}
]
},
{
"name": "va.openai.v2v.vision",
"auto_start": false,
"nodes": [
{
"type": "extension",
"extension_group": "rtc",
"addon": "agora_rtc",
"name": "agora_rtc",
"property": {
"app_id": "${env:AGORA_APP_ID}",
"token": "",
"channel": "ten_agent_test",
"stream_id": 1234,
"remote_stream_id": 123,
"subscribe_audio": true,
"subscribe_video": true,
"publish_audio": true,
"publish_data": true,
"subscribe_audio_sample_rate": 24000
}
},
{
"type": "extension",
"extension_group": "llm",
"addon": "openai_v2v_python",
"name": "openai_v2v_python",
"property": {
"api_key": "${env:OPENAI_REALTIME_API_KEY}",
"temperature": 0.9,
"model": "gpt-4o-realtime-preview",
"max_tokens": 2048,
"voice": "alloy",
"language": "en-US",
"server_vad": true,
"dump": true,
"history": 10
}
},
{
"type": "extension",
"extension_group": "transcriber",
"addon": "message_collector",
"name": "message_collector"
},
{
"type": "extension",
"extension_group": "tools",
"addon": "weatherapi_tool_python",
"name": "weatherapi_tool_python",
"property": {
"api_key": "${env:WEATHERAPI_API_KEY}"
}
},
{
"type": "extension",
"extension_group": "tools",
"addon": "bingsearch_tool_python",
"name": "bingsearch_tool_python",
"property": {
"api_key": "${env:BING_API_KEY}"
}
},
{
"type": "extension",
"extension_group": "tools",
"addon": "vision_tool_python",
"name": "vision_tool_python",
"property": {
"frequency_ms": 1000,
"use_llm": false
}
},
{
"type": "extension",
"extension_group": "azure_vision",
"addon": "azure_vision_python",
"name": "azure_vision_python",
"property": {
"key": "${env:AZURE_VISION_KEY}",
"endpoint": "https://tenagentvision.cognitiveservices.azure.com/"
}
}
],
"connections": [
{
"extension_group": "rtc",
"extension": "agora_rtc",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
],
"video_frame": [
{
"name": "video_frame",
"dest": [
{
"extension_group": "tools",
"extension": "vision_tool_python"
}
]
}
]
},
{
"extension_group": "tools",
"extension": "vision_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
},
{
"name": "image_analyze",
"dest": [
{
"extension_group": "azure_vision",
"extension": "azure_vision_python"
}
]
}
]
},
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
"extension_group": "tools",
"extension": "bingsearch_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
"extension_group": "llm",
"extension": "openai_v2v_python",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
}
],
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "transcriber",
"extension": "message_collector"
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
},
{
"name": "tool_call_get_current_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_past_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_future_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_bing_search",
"dest": [
{
"extension_group": "tools",
"extension": "bingsearch_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_query_single_image",
"dest": [
{
"extension_group": "tools",
"extension": "vision_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
}
]
},
{
"extension_group": "transcriber",
"extension": "message_collector",
"data": [
{
"name": "data",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
}
]
}
]
}
]
}
Expand Down
2 changes: 1 addition & 1 deletion demo/src/app/api/agents/start/graph.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ export const getGraphProperties = (
"azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType]
}
}
} else if (graphName == "va.openai.v2v") {
} else if (graphName == "va.openai.v2v" || graphName == "va.openai.v2v.vision") {
return {
"openai_v2v_python": {
"model": "gpt-4o-realtime-preview",
Expand Down
4 changes: 4 additions & 0 deletions demo/src/common/constant.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ export const GRAPH_OPTIONS: GraphOptionItem[] = [
{
label: "Voice Agent with OpenAI Realtime API (Beta)",
value: "va.openai.v2v"
},
{
label: "Voice Agent with OpenAI Realtime API (Beta) with Vision",
value: "va.openai.v2v.vision"
}
]

Expand Down

0 comments on commit 76e5ded

Please sign in to comment.