Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev/vision tool python #365

Open
wants to merge 12 commits into
base: main
Choose a base branch
from
+2,016 −35
Open

Dev/vision tool python #365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
50d6b51
vision is working.
tomasliu-agora Oct 14, 2024
bbcbc7d
Merge branch 'main' into dev/vision_tool_python
tomasliu-agora Oct 14, 2024
b0eb9bf
add azure vision
tomasliu-agora Oct 14, 2024
1e36a3d
fix readme and manifest
tomasliu-agora Oct 15, 2024
0a9b426
wip
tomasliu-agora Oct 16, 2024
da7a39f
update firestore tsdb
Oct 19, 2024
14a34e3
fix
Oct 19, 2024
4d00bdd
add firestore txn
Oct 19, 2024
7c910a4
abstract path name
Oct 19, 2024
5149dc6
Merge branch 'dev/tsdb' of github.com:KunkaiLin1205/TEN-Agent into de…
tomasliu-agora Oct 19, 2024
76e5ded
update graph
tomasliu-agora Oct 25, 2024
1169b4d
fix
tomasliu-agora Oct 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .env.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ ELEVENLABS_TTS_KEY=
# Gemini API key
GEMINI_API_KEY=

# Extension: azure_vision_python
# Azure vision key
AZURE_VISION_KEY=

# Extension: litellm
# Using Environment Variables, refer to https://docs.litellm.ai/docs/providers
# For example:
Expand Down
320 changes: 320 additions & 0 deletions agents/property.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2659,6 +2659,326 @@
]
}
]
},
{
"name": "va.openai.v2v.vision",
"auto_start": false,
"nodes": [
{
"type": "extension",
"extension_group": "rtc",
"addon": "agora_rtc",
"name": "agora_rtc",
"property": {
"app_id": "${env:AGORA_APP_ID}",
"token": "",
"channel": "ten_agent_test",
"stream_id": 1234,
"remote_stream_id": 123,
"subscribe_audio": true,
"subscribe_video": true,
"publish_audio": true,
"publish_data": true,
"subscribe_audio_sample_rate": 24000,
"subscribe_video_pix_fmt": 4
}
},
{
"type": "extension",
"extension_group": "llm",
"addon": "openai_v2v_python",
"name": "openai_v2v_python",
"property": {
"api_key": "${env:OPENAI_REALTIME_API_KEY}",
"temperature": 0.9,
"model": "gpt-4o-realtime-preview",
"max_tokens": 2048,
"voice": "alloy",
"language": "en-US",
"server_vad": true,
"dump": true,
"history": 10
}
},
{
"type": "extension",
"extension_group": "transcriber",
"addon": "message_collector",
"name": "message_collector"
},
{
"type": "extension",
"extension_group": "tools",
"addon": "weatherapi_tool_python",
"name": "weatherapi_tool_python",
"property": {
"api_key": "${env:WEATHERAPI_API_KEY}"
}
},
{
"type": "extension",
"extension_group": "tools",
"addon": "bingsearch_tool_python",
"name": "bingsearch_tool_python",
"property": {
"api_key": "${env:BING_API_KEY}"
}
},
{
"type": "extension",
"extension_group": "tools",
"addon": "vision_tool_python",
"name": "vision_tool_python",
"property": {
"frequency_ms": 1000,
"use_llm": false
}
},
{
"type": "extension",
"extension_group": "azure_vision",
"addon": "azure_vision_python",
"name": "azure_vision_python",
"property": {
"key": "${env:AZURE_VISION_KEY}",
"endpoint": "https://tenagentvision.cognitiveservices.azure.com/"
}
}
],
"connections": [
{
"extension_group": "rtc",
"extension": "agora_rtc",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
],
"video_frame": [
{
"name": "video_frame",
"dest": [
{
"extension_group": "tools",
"extension": "vision_tool_python"
}
]
}
]
},
{
"extension_group": "tools",
"extension": "vision_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
},
{
"name": "image_analyze",
"dest": [
{
"extension_group": "azure_vision",
"extension": "azure_vision_python"
}
]
}
]
},
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
"extension_group": "tools",
"extension": "bingsearch_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
"extension_group": "llm",
"extension": "openai_v2v_python",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
}
],
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "transcriber",
"extension": "message_collector"
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
},
{
"name": "tool_call_get_current_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_past_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_future_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_bing_search",
"dest": [
{
"extension_group": "tools",
"extension": "bingsearch_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_query_single_image",
"dest": [
{
"extension_group": "tools",
"extension": "vision_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
}
]
},
{
"extension_group": "transcriber",
"extension": "message_collector",
"data": [
{
"name": "data",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
}
]
}
]
}
]
}
Expand Down
21 changes: 21 additions & 0 deletions agents/ten_packages/extension/azure_vision_python/BUILD.gn
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#
#
# Agora Real Time Engagement
# Created by Wei Hu in 2022-11.
# Copyright (c) 2024 Agora IO. All rights reserved.
#
#
import("//build/feature/ten_package.gni")

ten_package("azure_vision_python") {
package_kind = "extension"

resources = [
"__init__.py",
"addon.py",
"extension.py",
"log.py",
"manifest.json",
"property.json",
]
}
Loading