Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: only install requirements from required python extensions to speed up use/build processing #453

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+204 −536
Open

feat: only install requirements from required python extensions to speed up use/build processing #453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d279616
fix: remove BUILD.gn for python extensions
wangyoucao577 Dec 1, 2024
16dc140
feat: only install requirements if extension exists in manifest
wangyoucao577 Dec 1, 2024
3aeff79
fix: remove llama index preload
wangyoucao577 Dec 1, 2024
417c5fb
feat: reset in repo python extension versions to 0.1.0
wangyoucao577 Dec 1, 2024
f76e0c1
feat: add necessary py extensions in manifest
wangyoucao577 Dec 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions agents/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ ten_packages/extension/azure_tts
ten_packages/extension/agora_sess_ctrl
ten_packages/extension/agora_rtm
ten_packages/extension/py_init_extension_cpp
ten_packages/extension/http_server_python
ten_packages/system/agora_rtc_sdk
ten_packages/system/azure_speech_sdk
ten_packages/system/nlohmann_json
Expand Down
45 changes: 45 additions & 0 deletions agents/examples/demo/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,51 @@
"type": "extension",
"name": "agora_rtm",
"version": "=0.3.0"
},
{
"type": "extension",
"name": "openai_chatgpt_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "openai_v2v_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "fish_audio_tts",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "message_collector",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "message_collector_rtm",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "vision_tool_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "vision_analyze_tool_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "weatherapi_tool_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "bingsearch_tool_python",
"version": "=0.1.0"
}
]
}
135 changes: 135 additions & 0 deletions agents/examples/experimental/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,141 @@
"type": "extension",
"name": "agora_rtm",
"version": "=0.3.0"
},
{
"type": "extension",
"name": "http_server_python",
"version": "0.7.0"
},
{
"type": "extension",
"name": "aliyun_analyticdb_vector_storage",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "aliyun_text_embedding",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "file_chunker",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "llama_index_chat_engine",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "bedrock_llm_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "bingsearch_tool_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "cartesia_tts",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "cosy_tts_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "deepgram_asr_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "elevenlabs_tts_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "fashionai",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "fish_audio_tts",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "gemini_llm_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "gemini_llm_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "interrupt_detector_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "message_collector",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "minimax_tts_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "minimax_v2v_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "openai_chatgpt_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "openai_v2v_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "polly_tts",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "qwen_llm_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "transcribe_asr_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "tsdb_firestore",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "vision_tool_python",
"version": "=0.1.0"
},
{
"type": "extension",
"name": "weatherapi_tool_python",
"version": "=0.1.0"
}
]
}
10 changes: 5 additions & 5 deletions agents/scripts/install_deps_and_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,11 @@ install_python_requirements() {
if [[ -d "ten_packages/extension" ]]; then
for extension in ten_packages/extension/*; do
if [[ -f "$extension/requirements.txt" ]]; then
pip install -r $extension/requirements.txt
local extension_name=$(basename $extension)
local exists_in_manifest=$(cat manifest.json | jq --arg ext_name "${extension_name}" 'any(.dependencies[].name == $ext_name ; .)' )
if [[ "$exists_in_manifest" == "true" ]]; then
pip install -r $extension/requirements.txt
fi
fi
done
fi
Expand All @@ -73,10 +77,6 @@ install_python_requirements() {
fi
done
fi

# pre-import llama-index as it cloud download additional resources during the first import
echo "pre-import python modules..."
python3.10 -c "import llama_index.core;"
}

build_go_app() {
Expand Down
2 changes: 1 addition & 1 deletion agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"type": "extension",
"name": "aliyun_analyticdb_vector_storage",
"version": "0.4.0",
"version": "0.1.0",
"dependencies": [
{
"type": "system",
Expand Down
2 changes: 1 addition & 1 deletion agents/ten_packages/extension/aliyun_text_embedding/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"type": "extension",
"name": "aliyun_text_embedding",
"version": "0.4.0",
"version": "0.1.0",
"dependencies": [
{
"type": "system",
Expand Down
2 changes: 1 addition & 1 deletion agents/ten_packages/extension/bedrock_llm_python/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"type": "extension",
"name": "bedrock_llm_python",
"version": "0.4.0",
"version": "0.1.0",
"dependencies": [
{
"type": "system",
Expand Down
21 changes: 0 additions & 21 deletions agents/ten_packages/extension/bingsearch_tool_python/BUILD.gn
View file
Open in desktop

This file was deleted.

19 changes: 0 additions & 19 deletions agents/ten_packages/extension/cartesia_tts/BUILD.gn
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion agents/ten_packages/extension/cartesia_tts/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"type": "extension",
"name": "cartesia_tts",
"version": "0.4.2",
"version": "0.1.0",
"dependencies": [
{
"type": "system",
Expand Down
19 changes: 0 additions & 19 deletions agents/ten_packages/extension/cosy_tts_python/BUILD.gn
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion agents/ten_packages/extension/cosy_tts_python/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"type": "extension",
"name": "cosy_tts_python",
"version": "0.4.2",
"version": "0.1.0",
"dependencies": [
{
"type": "system",
Expand Down
19 changes: 0 additions & 19 deletions agents/ten_packages/extension/elevenlabs_tts_python/BUILD.gn
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion agents/ten_packages/extension/elevenlabs_tts_python/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"type": "extension",
"name": "elevenlabs_tts_python",
"version": "0.4.2",
"version": "0.1.0",
"dependencies": [
{
"type": "system",
Expand Down
22 changes: 0 additions & 22 deletions agents/ten_packages/extension/fashionai/BUILD.gn
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion agents/ten_packages/extension/file_chunker/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"type": "extension",
"name": "file_chunker",
"version": "0.4.0",
"version": "0.1.0",
"dependencies": [
{
"type": "system",
Expand Down
Loading