diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e899eb94..bf7a4c1c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,11 +2,12 @@ // README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile { "name": "astra", - "image": "ghcr.io/ten-framework/astra_agents_build:0.4.0", + "image": "ghcr.io/ten-framework/ten_agent_build:0.1.0", "customizations": { "vscode": { "extensions": [ - "golang.go" + "golang.go", + "ms-vscode.cpptools" ] } }, @@ -19,6 +20,7 @@ ], // Features to add to the dev container. More info: https://containers.dev/features. "features": { - "ghcr.io/devcontainers/features/git:1": {} + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/python:1": {} } } \ No newline at end of file diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index df45ecc6..68f59b50 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -1,16 +1,16 @@ name: Build Docker -on: +on: push: - branches: [ "main" ] + branches: ["main"] # Publish semver tags as releases. - tags: [ 'v*.*.*' ] + tags: ["v*.*.*"] paths-ignore: - - '.devcontainer/**' - - '.github/ISSUE_TEMPLATE/**' - - 'images/**' - - 'playground/**' - - '**.md' + - ".devcontainer/**" + - ".github/ISSUE_TEMPLATE/**" + - "images/**" + - "playground/**" + - "**.md" pull_request: branches: [ "main" ] workflow_dispatch: @@ -18,36 +18,46 @@ on: env: SERVER_IMAGE_NAME: astra_agents_server PLAYGROUND_IMAGE_NAME: astra_playground + DEMO_IMAGE_NAME: agent_demo jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-tags: true - fetch-depth: '0' - - id: pre-step - shell: bash - run: echo "image-tag=$(git describe --tags --always)" >> $GITHUB_OUTPUT - - name: Build & Publish Docker Image for Agents Server - uses: elgohr/Publish-Docker-Github-Action@v5 - with: - name: ${{ github.repository_owner }}/${{ env.SERVER_IMAGE_NAME }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ghcr.io - tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" - no_push: ${{ github.event_name == 'pull_request' }} - - name: Build & Publish Docker Image for Playground - uses: elgohr/Publish-Docker-Github-Action@v5 - with: - name: ${{ github.repository_owner }}/${{ env.PLAYGROUND_IMAGE_NAME }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ghcr.io - workdir: playground - tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" - no_push: ${{ github.event_name == 'pull_request' }} - + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-tags: true + fetch-depth: "0" + - id: pre-step + shell: bash + run: echo "image-tag=$(git describe --tags --always)" >> $GITHUB_OUTPUT + - name: Build & Publish Docker Image for Agents Server + uses: elgohr/Publish-Docker-Github-Action@v5 + with: + name: ${{ github.repository_owner }}/${{ env.SERVER_IMAGE_NAME }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" + no_push: ${{ github.event_name == 'pull_request' }} + - name: Build & Publish Docker Image for Playground + uses: elgohr/Publish-Docker-Github-Action@v5 + with: + name: ${{ github.repository_owner }}/${{ env.PLAYGROUND_IMAGE_NAME }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + workdir: playground + tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" + no_push: ${{ github.event_name == 'pull_request' }} + - name: Build & Publish Docker Image for demo + uses: elgohr/Publish-Docker-Github-Action@v5 + with: + name: ${{ github.repository_owner }}/${{ env.DEMO_IMAGE_NAME }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + workdir: demo + tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" + no_push: ${{ github.event_name == 'pull_request' }} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..8795a5f2 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,47 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "debug go", + "type": "go", + "request": "launch", + "mode": "exec", + "cwd": "${workspaceFolder}", + "program": "${workspaceFolder}/agents/bin/worker", + "env": { + "LD_LIBRARY_PATH": "${workspaceFolder}/agents/ten_packages/system/ten_runtime_go/lib:${workspaceFolder}/agents/ten_packages/system/agora_rtc_sdk/lib:${workspaceFolder}/agents/ten_packages/system/azure_speech_sdk/lib", + "TEN_APP_BASE_DIR": "${workspaceFolder}/agents" + } + }, + { + "name": "debug python", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "preLaunchTask": "start app" + }, + { + "name": "debug cpp", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/agents/bin/worker", + "cwd": "${workspaceFolder}", + "environment": [ + { + "name": "LD_LIBRARY_PATH", + "value": "${workspaceFolder}/agents/ten_packages/system/agora_rtc_sdk/lib:${workspaceFolder}/agents/ten_packages/system/azure_speech_sdk/lib" + }, + { + "name": "CGO_LDFLAGS", + "value": "-L${workspaceFolder}/agents/ten_packages/system/ten_runtime_go/lib -lten_runtime_go -Wl,-rpath,@loader_path/lib -Wl,-rpath,@loader_path/../lib" + } + ] + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000..da11248b --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,22 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "shell", + "label": "build", + "command": "make build", + "args": [], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "start app", + "type": "shell", + "command": "export TEN_ENABLE_PYTHON_DEBUG=true; export TEN_PYTHON_DEBUG_PORT=5678; ./agents/bin/start", + "group": "none", + "isBackground": true + }, + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 77792a4c..97adf23a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ten-framework/astra_agents_build:0.4.0 AS builder +FROM ghcr.io/ten-framework/ten_agent_build:0.1.0 AS builder ARG SESSION_CONTROL_CONF=session_control.conf diff --git a/agents/main.go b/agents/main.go index 61107505..b42101f5 100644 --- a/agents/main.go +++ b/agents/main.go @@ -50,7 +50,6 @@ func startAppBlocking(cfg *appConfig) { appInstance.Run(true) appInstance.Wait() - ten.UnloadAllAddons() ten.EnsureCleanupWhenProcessExit() } diff --git a/agents/manifest-lock.json b/agents/manifest-lock.json index 09b454c8..902dca63 100644 --- a/agents/manifest-lock.json +++ b/agents/manifest-lock.json @@ -1,10 +1,11 @@ { + "version": 1, "packages": [ { "type": "system", "name": "ten_runtime_go", - "version": "0.1.0", - "hash": "ab66a8ed40c744a52cce36f26a233e669d989d9f620876156e3cc7187f214977", + "version": "0.2.0", + "hash": "8b582de5dfaa38983104143fbb6c530b3aeb463ad9e9ef51f2c72fba9862b8cc", "dependencies": [ { "type": "system", @@ -21,8 +22,8 @@ { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.1.0", - "hash": "b39c4dddbec58e1a756b7e71f41ab0ec419ab0043525ba94e6ed98b7ef634697", + "version": "0.2.0", + "hash": "e1858dfd83d18a69901cefb2edfd52e57ee326a3d306e799ff1d661f3195bb6b", "dependencies": [ { "type": "system", @@ -43,8 +44,8 @@ { "type": "extension_group", "name": "default_extension_group", - "version": "0.1.0", - "hash": "cfadaf8f951de42965e92becc67a597501196bc3bd6f17f39a64260836393c64", + "version": "0.2.0", + "hash": "117ed3e747654fc1282129a160eaecc2cd16548e70aa22128efee21f10e185c8", "dependencies": [ { "type": "system", @@ -61,8 +62,8 @@ { "type": "extension", "name": "agora_rtc", - "version": "0.5.1", - "hash": "de8bb179155f6a329072be87ccd624177e9bbe795956a9a275e7886a3dc31cb7", + "version": "0.7.0-rc2", + "hash": "89d7af8f84d06afbd79901e0057182280f3f430227ad6fae98ec154067ffa82c", "dependencies": [ { "type": "system", @@ -72,10 +73,6 @@ "type": "system", "name": "agora_rtc_sdk" }, - { - "type": "system", - "name": "azure_speech_sdk" - }, { "type": "system", "name": "nlohmann_json" @@ -88,11 +85,23 @@ } ] }, + { + "type": "system", + "name": "azure_speech_sdk", + "version": "1.38.0", + "hash": "66a50ef361f8190fa0595d8298c135e13b73796d57174a0802631263a8f15806", + "supports": [ + { + "os": "linux", + "arch": "x64" + } + ] + }, { "type": "extension", "name": "azure_tts", "version": "0.4.0", - "hash": "4f25e8c2a9c82f2a699a4e8378d11d2fbb31c90ec5df1718271ebfe6377a5389", + "hash": "c8f838754aaae7ed4598e99fb94b6e251382ade08fa23dbf85e91e9864d018ef", "dependencies": [ { "type": "system", @@ -107,20 +116,8 @@ { "type": "system", "name": "ten_runtime", - "version": "0.1.0", - "hash": "b630f52ef9787132dc854fb4e90f962336be3f836baba137ca3b6dc132df0b86", - "supports": [ - { - "os": "linux", - "arch": "x64" - } - ] - }, - { - "type": "system", - "name": "azure_speech_sdk", - "version": "1.38.0", - "hash": "66a50ef361f8190fa0595d8298c135e13b73796d57174a0802631263a8f15806", + "version": "0.2.0", + "hash": "7effdb036d5bf91894060a9230775ff8ec2598f202b8238578f99a14dbf11632", "supports": [ { "os": "linux", @@ -131,8 +128,8 @@ { "type": "system", "name": "agora_rtc_sdk", - "version": "4.1.35+build328115", - "hash": "fd33989f9913d77e05970eb2b265fa5e08322141b7e0f8f9fd3e521f87929b3b", + "version": "4.1.36+build331418", + "hash": "74115dd35822dc3b09fbadb577b31146af704d60435ca401c0b305cce80b2ba4", "supports": [ { "os": "linux", @@ -149,8 +146,8 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0", - "hash": "a8980c39ba0cf1f21b38490c3167950f750403b1b29d756cfbacc5c5147becd7", + "version": "0.2.0", + "hash": "b44d3767f364583f8bb8e8995f6f7f49e3af27b3c9a8ddf62fa319f1fc39910e", "dependencies": [ { "type": "system", diff --git a/agents/manifest.json b/agents/manifest.json index 4dfb524e..4ba7b513 100644 --- a/agents/manifest.json +++ b/agents/manifest.json @@ -6,22 +6,27 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.1" + "version": "0.2" }, { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.1" + "version": "0.2" }, { "type": "extension_group", "name": "default_extension_group", - "version": "0.1" + "version": "0.2" }, { "type": "extension", "name": "agora_rtc", - "version": "=0.5.1" + "version": "=0.7.0-rc2" + }, + { + "type": "system", + "name": "azure_speech_sdk", + "version": "1.38.0" }, { "type": "extension", diff --git a/agents/property.json b/agents/property.json index 880241c7..c21b4baf 100644 --- a/agents/property.json +++ b/agents/property.json @@ -4,7 +4,7 @@ "predefined_graphs": [ { "name": "va.openai.azure", - "auto_start": true, + "auto_start": false, "nodes": [ { "type": "extension", @@ -12,7 +12,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -23,8 +23,8 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf" } }, @@ -41,7 +41,7 @@ "name": "openai_chatgpt", "property": { "base_url": "", - "api_key": "$OPENAI_API_KEY", + "api_key": "${env:OPENAI_API_KEY}", "frequency_penalty": 0.9, "model": "gpt-4o-mini", "max_tokens": 512, @@ -57,8 +57,8 @@ "addon": "azure_tts", "name": "azure_tts", "property": { - "azure_subscription_key": "$AZURE_TTS_KEY", - "azure_subscription_region": "$AZURE_TTS_REGION", + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", "azure_synthesis_voice_name": "en-US-JaneNeural" } }, @@ -211,7 +211,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -222,8 +222,8 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf" } }, @@ -240,7 +240,7 @@ "name": "openai_chatgpt", "property": { "base_url": "", - "api_key": "$OPENAI_API_KEY", + "api_key": "${env:OPENAI_API_KEY}", "frequency_penalty": 0.9, "model": "gpt-4o-mini", "max_tokens": 512, @@ -256,7 +256,7 @@ "addon": "elevenlabs_tts", "name": "elevenlabs_tts", "property": { - "api_key": "$ELEVENLABS_TTS_KEY", + "api_key": "${env:ELEVENLABS_TTS_KEY}", "model_id": "eleven_multilingual_v2", "optimize_streaming_latency": 0, "request_timeout_seconds": 30, @@ -416,7 +416,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -427,8 +427,8 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf" } }, @@ -439,8 +439,8 @@ "name": "bedrock_llm", "property": { "region": "us-east-1", - "access_key": "$AWS_ACCESS_KEY_ID", - "secret_key": "$AWS_SECRET_ACCESS_KEY", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", "max_tokens": 512, "prompt": "", @@ -454,8 +454,8 @@ "addon": "azure_tts", "name": "azure_tts", "property": { - "azure_subscription_key": "$AZURE_TTS_KEY", - "azure_subscription_region": "$AZURE_TTS_REGION", + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", "azure_synthesis_voice_name": "en-US-JaneNeural" } }, @@ -580,7 +580,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -591,8 +591,8 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf" } }, @@ -603,7 +603,7 @@ "name": "openai_chatgpt", "property": { "base_url": "", - "api_key": "$OPENAI_API_KEY", + "api_key": "${env:OPENAI_API_KEY}", "frequency_penalty": 0.9, "model": "gpt-4o-mini", "max_tokens": 512, @@ -619,7 +619,7 @@ "addon": "cosy_tts", "name": "cosy_tts", "property": { - "api_key": "$QWEN_API_KEY", + "api_key": "${env:QWEN_API_KEY}", "model": "cosyvoice-v1", "voice": "longxiaochun", "sample_rate": 16000 @@ -780,7 +780,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -791,8 +791,8 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf" } }, @@ -802,7 +802,7 @@ "addon": "qwen_llm_python", "name": "qwen_llm", "property": { - "api_key": "$QWEN_API_KEY", + "api_key": "${env:QWEN_API_KEY}", "model": "qwen-max", "max_tokens": 512, "prompt": "", @@ -815,7 +815,7 @@ "addon": "cosy_tts", "name": "cosy_tts", "property": { - "api_key": "$QWEN_API_KEY", + "api_key": "${env:QWEN_API_KEY}", "model": "cosyvoice-v1", "voice": "longxiaochun", "sample_rate": 16000 @@ -983,7 +983,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -994,8 +994,8 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf" } }, @@ -1006,8 +1006,8 @@ "name": "bedrock_llm", "property": { "region": "us-east-1", - "access_key": "$AWS_ACCESS_KEY_ID", - "secret_key": "$AWS_SECRET_ACCESS_KEY", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", "max_tokens": 512, "prompt": "", @@ -1022,8 +1022,8 @@ "name": "polly_tts", "property": { "region": "us-east-1", - "access_key": "$AWS_ACCESS_KEY_ID", - "secret_key": "$AWS_SECRET_ACCESS_KEY", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", "engine": "generative", "voice": "Ruth", "sample_rate": "16000", @@ -1151,7 +1151,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -1162,8 +1162,8 @@ "enable_agora_asr": false, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf" } }, @@ -1174,8 +1174,8 @@ "name": "transcribe_asr", "property": { "region": "us-east-1", - "access_key": "$AWS_ACCESS_KEY_ID", - "secret_key": "$AWS_SECRET_ACCESS_KEY", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", "sample_rate": "16000", "lang_code": "en-US" } @@ -1187,8 +1187,8 @@ "name": "bedrock_llm", "property": { "region": "us-east-1", - "access_key": "$AWS_ACCESS_KEY_ID", - "secret_key": "$AWS_SECRET_ACCESS_KEY", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", "max_tokens": 512, "prompt": "", @@ -1203,8 +1203,8 @@ "name": "polly_tts", "property": { "region": "us-east-1", - "access_key": "$AWS_ACCESS_KEY_ID", - "secret_key": "$AWS_SECRET_ACCESS_KEY", + "access_key": "${env:AWS_ACCESS_KEY_ID}", + "secret_key": "${env:AWS_SECRET_ACCESS_KEY}", "engine": "generative", "voice": "Ruth", "sample_rate": "16000", @@ -1381,7 +1381,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -1393,8 +1393,8 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf", "subscribe_video_pix_fmt": 4 } @@ -1412,7 +1412,7 @@ "name": "openai_chatgpt", "property": { "base_url": "", - "api_key": "$OPENAI_API_KEY", + "api_key": "${env:OPENAI_API_KEY}", "frequency_penalty": 0.9, "model": "gpt-4o", "max_tokens": 512, @@ -1430,8 +1430,8 @@ "addon": "azure_tts", "name": "azure_tts", "property": { - "azure_subscription_key": "$AZURE_TTS_KEY", - "azure_subscription_region": "$AZURE_TTS_REGION", + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", "azure_synthesis_voice_name": "en-US-JaneNeural" } }, @@ -1587,7 +1587,7 @@ }, { "name": "va.gemini.azure", - "auto_start": true, + "auto_start": false, "nodes": [ { "type": "extension", @@ -1595,7 +1595,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -1606,8 +1606,8 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf" } }, @@ -1640,8 +1640,8 @@ "addon": "azure_tts", "name": "azure_tts", "property": { - "azure_subscription_key": "$AZURE_TTS_KEY", - "azure_subscription_region": "$AZURE_TTS_REGION", + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", "azure_synthesis_voice_name": "en-US-JaneNeural" } }, @@ -1786,7 +1786,7 @@ }, { "name": "va.qwen.rag", - "auto_start": true, + "auto_start": false, "nodes": [ { "type": "extension", @@ -1794,7 +1794,7 @@ "addon": "agora_rtc", "name": "agora_rtc", "property": { - "app_id": "$AGORA_APP_ID", + "app_id": "${env:AGORA_APP_ID}", "token": "", "channel": "astra_agents_test", "stream_id": 1234, @@ -1805,8 +1805,8 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "$AZURE_STT_KEY", - "agora_asr_vendor_region": "$AZURE_STT_REGION", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", "agora_asr_session_control_file_path": "session_control.conf" } }, @@ -1816,7 +1816,7 @@ "addon": "qwen_llm_python", "name": "qwen_llm", "property": { - "api_key": "$QWEN_API_KEY", + "api_key": "${env:QWEN_API_KEY}", "model": "qwen-max", "max_tokens": 512, "prompt": "", @@ -1829,7 +1829,7 @@ "addon": "cosy_tts", "name": "cosy_tts", "property": { - "api_key": "$QWEN_API_KEY", + "api_key": "${env:QWEN_API_KEY}", "model": "cosyvoice-v1", "voice": "longxiaochun", "sample_rate": 16000 @@ -1841,8 +1841,8 @@ "addon": "azure_tts", "name": "azure_tts", "property": { - "azure_subscription_key": "$AZURE_TTS_KEY", - "azure_subscription_region": "$AZURE_TTS_REGION", + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", "azure_synthesis_voice_name": "en-US-JaneNeural" } }, @@ -1874,7 +1874,7 @@ "addon": "aliyun_text_embedding", "name": "aliyun_text_embedding", "property": { - "api_key": "$ALIYUN_TEXT_EMBEDDING_API_KEY", + "api_key": "${env:ALIYUN_TEXT_EMBEDDING_API_KEY}", "model": "text-embedding-v3" } }, @@ -1884,14 +1884,14 @@ "addon": "aliyun_analyticdb_vector_storage", "name": "aliyun_analyticdb_vector_storage", "property": { - "alibaba_cloud_access_key_id": "$ALIBABA_CLOUD_ACCESS_KEY_ID", - "alibaba_cloud_access_key_secret": "$ALIBABA_CLOUD_ACCESS_KEY_SECRET", - "adbpg_instance_id": "$ALIYUN_ANALYTICDB_INSTANCE_ID", - "adbpg_instance_region": "$ALIYUN_ANALYTICDB_INSTANCE_REGION", - "adbpg_account": "$ALIYUN_ANALYTICDB_ACCOUNT", - "adbpg_account_password": "$ALIYUN_ANALYTICDB_ACCOUNT_PASSWORD", - "adbpg_namespace": "$ALIYUN_ANALYTICDB_NAMESPACE", - "adbpg_namespace_password": "$ALIYUN_ANALYTICDB_NAMESPACE_PASSWORD" + "alibaba_cloud_access_key_id": "${env:ALIBABA_CLOUD_ACCESS_KEY_ID}", + "alibaba_cloud_access_key_secret": "${env:ALIBABA_CLOUD_ACCESS_KEY_SECRET}", + "adbpg_instance_id": "${env:ALIYUN_ANALYTICDB_INSTANCE_ID}", + "adbpg_instance_region": "${env:ALIYUN_ANALYTICDB_INSTANCE_REGION}", + "adbpg_account": "${env:ALIYUN_ANALYTICDB_ACCOUNT}", + "adbpg_account_password": "${env:ALIYUN_ANALYTICDB_ACCOUNT_PASSWORD}", + "adbpg_namespace": "${env:ALIYUN_ANALYTICDB_NAMESPACE}", + "adbpg_namespace_password": "${env:ALIYUN_ANALYTICDB_NAMESPACE_PASSWORD}" } }, { diff --git a/agents/scripts/install_deps_and_build.sh b/agents/scripts/install_deps_and_build.sh index 4d1538ff..9c5977cd 100755 --- a/agents/scripts/install_deps_and_build.sh +++ b/agents/scripts/install_deps_and_build.sh @@ -6,6 +6,9 @@ OS="linux" # x64, arm64 CPU="x64" +# debug, release +BUILD_TYPE="release" + build_cxx_extensions() { local app_dir=$1 @@ -16,8 +19,8 @@ build_cxx_extensions() { cp $app_dir/scripts/BUILD.gn $app_dir - tgn gen $OS $CPU release -- is_clang=false - tgn build $OS $CPU release + tgn gen $OS $CPU $BUILD_TYPE -- is_clang=false enable_sanitizer=false + tgn build $OS $CPU $BUILD_TYPE local ret=$? diff --git a/agents/ten_packages/bak/openai_chatgpt_python/__init__.py b/agents/ten_packages/bak/openai_chatgpt_python/__init__.py new file mode 100644 index 00000000..42c4cd12 --- /dev/null +++ b/agents/ten_packages/bak/openai_chatgpt_python/__init__.py @@ -0,0 +1,4 @@ +from . import openai_chatgpt_addon +from .log import logger + +logger.info("openai_chatgpt_python extension loaded") diff --git a/agents/ten_packages/bak/openai_chatgpt_python/log.py b/agents/ten_packages/bak/openai_chatgpt_python/log.py new file mode 100644 index 00000000..fa2202da --- /dev/null +++ b/agents/ten_packages/bak/openai_chatgpt_python/log.py @@ -0,0 +1,13 @@ +import logging + +logger = logging.getLogger("openai_chatgpt_python") +logger.setLevel(logging.INFO) + +formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - [%(filename)s:%(lineno)d] - %(message)s" +) + +console_handler = logging.StreamHandler() +console_handler.setFormatter(formatter) + +logger.addHandler(console_handler) diff --git a/agents/ten_packages/bak/openai_chatgpt_python/manifest.json b/agents/ten_packages/bak/openai_chatgpt_python/manifest.json new file mode 100644 index 00000000..ce872dfe --- /dev/null +++ b/agents/ten_packages/bak/openai_chatgpt_python/manifest.json @@ -0,0 +1,93 @@ +{ + "type": "extension", + "name": "openai_chatgpt_python", + "version": "0.4.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.2" + } + ], + "api": { + "property": { + "api_key": { + "type": "string" + }, + "frequency_penalty": { + "type": "float64" + }, + "presence_penalty": { + "type": "float64" + }, + "temperature": { + "type": "float64" + }, + "top_p": { + "type": "float64" + }, + "model": { + "type": "string" + }, + "max_tokens": { + "type": "int64" + }, + "base_url": { + "type": "string" + }, + "prompt": { + "type": "string" + }, + "greeting": { + "type": "string" + }, + "checking_vision_text_items": { + "type": "string" + }, + "proxy_url": { + "type": "string" + }, + "max_memory_length": { + "type": "int64" + }, + "enable_tools": { + "type": "bool" + } + }, + "data_in": [ + { + "name": "text_data", + "property": { + "text": { + "type": "string" + } + } + } + ], + "data_out": [ + { + "name": "text_data", + "property": { + "text": { + "type": "string" + } + } + } + ], + "cmd_in": [ + { + "name": "flush" + } + ], + "cmd_out": [ + { + "name": "flush" + } + ], + "video_frame_in": [ + { + "name": "video_frame" + } + ] + } +} \ No newline at end of file diff --git a/agents/ten_packages/extension/openai_chatgpt_python/openai_chatgpt.py b/agents/ten_packages/bak/openai_chatgpt_python/openai_chatgpt.py similarity index 100% rename from agents/ten_packages/extension/openai_chatgpt_python/openai_chatgpt.py rename to agents/ten_packages/bak/openai_chatgpt_python/openai_chatgpt.py diff --git a/agents/ten_packages/extension/openai_chatgpt_python/openai_chatgpt_addon.py b/agents/ten_packages/bak/openai_chatgpt_python/openai_chatgpt_addon.py similarity index 100% rename from agents/ten_packages/extension/openai_chatgpt_python/openai_chatgpt_addon.py rename to agents/ten_packages/bak/openai_chatgpt_python/openai_chatgpt_addon.py diff --git a/agents/ten_packages/extension/openai_chatgpt_python/openai_chatgpt_extension.py b/agents/ten_packages/bak/openai_chatgpt_python/openai_chatgpt_extension.py similarity index 100% rename from agents/ten_packages/extension/openai_chatgpt_python/openai_chatgpt_extension.py rename to agents/ten_packages/bak/openai_chatgpt_python/openai_chatgpt_extension.py diff --git a/agents/ten_packages/bak/openai_chatgpt_python/property.json b/agents/ten_packages/bak/openai_chatgpt_python/property.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/agents/ten_packages/bak/openai_chatgpt_python/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/agents/ten_packages/bak/openai_chatgpt_python/requirements.txt b/agents/ten_packages/bak/openai_chatgpt_python/requirements.txt new file mode 100644 index 00000000..5ddef5be --- /dev/null +++ b/agents/ten_packages/bak/openai_chatgpt_python/requirements.txt @@ -0,0 +1,5 @@ +openai +numpy +requests +pillow +asyncio \ No newline at end of file diff --git a/agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json b/agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json index 4137a8f2..d5dd00f9 100644 --- a/agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json +++ b/agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/aliyun_text_embedding/manifest.json b/agents/ten_packages/extension/aliyun_text_embedding/manifest.json index 627c044f..ea02f9e0 100644 --- a/agents/ten_packages/extension/aliyun_text_embedding/manifest.json +++ b/agents/ten_packages/extension/aliyun_text_embedding/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/azure_tts/manifest.json b/agents/ten_packages/extension/azure_tts/manifest.json index 4f9fdfe1..8c7b8e31 100644 --- a/agents/ten_packages/extension/azure_tts/manifest.json +++ b/agents/ten_packages/extension/azure_tts/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.1" + "version": "0.2" }, { "type": "system", diff --git a/agents/ten_packages/extension/bedrock_llm_python/manifest.json b/agents/ten_packages/extension/bedrock_llm_python/manifest.json index 602e5a84..7a122b9e 100644 --- a/agents/ten_packages/extension/bedrock_llm_python/manifest.json +++ b/agents/ten_packages/extension/bedrock_llm_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/chat_transcriber/manifest.json b/agents/ten_packages/extension/chat_transcriber/manifest.json index 0d03a858..c318eba2 100644 --- a/agents/ten_packages/extension/chat_transcriber/manifest.json +++ b/agents/ten_packages/extension/chat_transcriber/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/chat_transcriber_python/manifest.json b/agents/ten_packages/extension/chat_transcriber_python/manifest.json index ad8c8bde..f56247ba 100644 --- a/agents/ten_packages/extension/chat_transcriber_python/manifest.json +++ b/agents/ten_packages/extension/chat_transcriber_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/cosy_tts/manifest.json b/agents/ten_packages/extension/cosy_tts/manifest.json index 4b1c55bc..24ac6b09 100644 --- a/agents/ten_packages/extension/cosy_tts/manifest.json +++ b/agents/ten_packages/extension/cosy_tts/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/elevenlabs_tts/manifest.json b/agents/ten_packages/extension/elevenlabs_tts/manifest.json index 25fa61e3..b3fcc92f 100644 --- a/agents/ten_packages/extension/elevenlabs_tts/manifest.json +++ b/agents/ten_packages/extension/elevenlabs_tts/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/elevenlabs_tts_python/manifest.json b/agents/ten_packages/extension/elevenlabs_tts_python/manifest.json index 48a956b1..7c0cc442 100644 --- a/agents/ten_packages/extension/elevenlabs_tts_python/manifest.json +++ b/agents/ten_packages/extension/elevenlabs_tts_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/file_chunker/manifest.json b/agents/ten_packages/extension/file_chunker/manifest.json index 02e91d2f..1416c357 100644 --- a/agents/ten_packages/extension/file_chunker/manifest.json +++ b/agents/ten_packages/extension/file_chunker/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/gemini_llm_python/manifest.json b/agents/ten_packages/extension/gemini_llm_python/manifest.json index d652b105..10f8f6ad 100644 --- a/agents/ten_packages/extension/gemini_llm_python/manifest.json +++ b/agents/ten_packages/extension/gemini_llm_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/http_server_python/manifest.json b/agents/ten_packages/extension/http_server_python/manifest.json index 05f62097..e770edda 100644 --- a/agents/ten_packages/extension/http_server_python/manifest.json +++ b/agents/ten_packages/extension/http_server_python/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "http_server_python", - "version": "0.4.0", + "version": "0.5.0", "dependencies": [ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "package": { diff --git a/agents/ten_packages/extension/interrupt_detector/manifest.json b/agents/ten_packages/extension/interrupt_detector/manifest.json index 7e4beda6..feb17b2c 100644 --- a/agents/ten_packages/extension/interrupt_detector/manifest.json +++ b/agents/ten_packages/extension/interrupt_detector/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/interrupt_detector_python/manifest.json b/agents/ten_packages/extension/interrupt_detector_python/manifest.json index bf872358..97e540ad 100644 --- a/agents/ten_packages/extension/interrupt_detector_python/manifest.json +++ b/agents/ten_packages/extension/interrupt_detector_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/llama_index_chat_engine/manifest.json b/agents/ten_packages/extension/llama_index_chat_engine/manifest.json index 0feacf83..622d24fa 100644 --- a/agents/ten_packages/extension/llama_index_chat_engine/manifest.json +++ b/agents/ten_packages/extension/llama_index_chat_engine/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/message_collector/manifest.json b/agents/ten_packages/extension/message_collector/manifest.json index c4004695..655a37a3 100644 --- a/agents/ten_packages/extension/message_collector/manifest.json +++ b/agents/ten_packages/extension/message_collector/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.2" } ], "package": { diff --git a/agents/ten_packages/extension/message_collector/src/extension.py b/agents/ten_packages/extension/message_collector/src/extension.py index 39206a13..7013c432 100644 --- a/agents/ten_packages/extension/message_collector/src/extension.py +++ b/agents/ten_packages/extension/message_collector/src/extension.py @@ -7,6 +7,7 @@ # import json import time +import uuid from ten import ( AudioFrame, VideoFrame, @@ -19,7 +20,8 @@ ) from .log import logger - +MAX_SIZE = 800 # 1 KB limit +OVERHEAD_ESTIMATE = 200 # Estimate for the overhead of metadata in the JSON CMD_NAME_FLUSH = "flush" @@ -89,16 +91,12 @@ def on_data(self, ten_env: TenEnv, data: Data) -> None: try: final = data.get_property_bool(TEXT_DATA_FINAL_FIELD) except Exception as e: - logger.warning( - f"on_data get_property_bool {TEXT_DATA_FINAL_FIELD} error: {e}" - ) + pass try: stream_id = data.get_property_int(TEXT_DATA_STREAM_ID_FIELD) except Exception as e: - logger.warning( - f"on_data get_property_int {TEXT_DATA_STREAM_ID_FIELD} error: {e}" - ) + pass try: end_of_segment = data.get_property_bool(TEXT_DATA_END_OF_SEGMENT_FIELD) @@ -124,19 +122,72 @@ def on_data(self, ten_env: TenEnv, data: Data) -> None: cached_text_map[stream_id] = text - msg_data = json.dumps({ - "text": text, + # Generate a unique message ID for this batch of parts + message_id = str(uuid.uuid4()) + + # Prepare the main JSON structure without the text field + base_msg_data = { "is_final": end_of_segment, "stream_id": stream_id, + "message_id": message_id, # Add message_id to identify the split message "data_type": "transcribe", "text_ts": int(time.time() * 1000), # Convert to milliseconds - }) + } try: - # convert the origin text data to the protobuf data and send it to the graph. - ten_data = Data.create("data") - ten_data.set_property_buf("data", msg_data.encode()) - ten_env.send_data(ten_data) + # Convert the text to UTF-8 bytes + text_bytes = text.encode('utf-8') + + # If the text + metadata fits within the size limit, send it directly + if len(text_bytes) + OVERHEAD_ESTIMATE <= MAX_SIZE: + base_msg_data["text"] = text + msg_data = json.dumps(base_msg_data) + ten_data = Data.create("data") + ten_data.set_property_buf("data", msg_data.encode()) + ten_env.send_data(ten_data) + else: + # Split the text bytes into smaller chunks, ensuring safe UTF-8 splitting + max_text_size = MAX_SIZE - OVERHEAD_ESTIMATE + total_length = len(text_bytes) + total_parts = (total_length + max_text_size - 1) // max_text_size # Calculate number of parts + + def get_valid_utf8_chunk(start, end): + """Helper function to ensure valid UTF-8 chunks.""" + while end > start: + try: + # Decode to check if this chunk is valid UTF-8 + text_part = text_bytes[start:end].decode('utf-8') + return text_part, end + except UnicodeDecodeError: + # Reduce the end point to avoid splitting in the middle of a character + end -= 1 + # If no valid chunk is found (shouldn't happen with valid UTF-8 input), return an empty string + return "", start + + part_number = 0 + start_index = 0 + while start_index < total_length: + part_number += 1 + # Get a valid UTF-8 chunk + text_part, end_index = get_valid_utf8_chunk(start_index, min(start_index + max_text_size, total_length)) + + # Prepare the part data with metadata + part_data = base_msg_data.copy() + part_data.update({ + "text": text_part, + "part_number": part_number, + "total_parts": total_parts, + }) + + # Send each part + part_msg_data = json.dumps(part_data) + ten_data = Data.create("data") + ten_data.set_property_buf("data", part_msg_data.encode()) + ten_env.send_data(ten_data) + + # Move to the next chunk + start_index = end_index + except Exception as e: logger.warning(f"on_data new_data error: {e}") return diff --git a/agents/ten_packages/extension/openai_chatgpt/manifest.json b/agents/ten_packages/extension/openai_chatgpt/manifest.json index 4bc6783e..a0f97290 100644 --- a/agents/ten_packages/extension/openai_chatgpt/manifest.json +++ b/agents/ten_packages/extension/openai_chatgpt/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/openai_chatgpt_python/BUILD.gn b/agents/ten_packages/extension/openai_chatgpt_python/BUILD.gn new file mode 100644 index 00000000..23f06108 --- /dev/null +++ b/agents/ten_packages/extension/openai_chatgpt_python/BUILD.gn @@ -0,0 +1,21 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2022-11. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +import("//build/feature/ten_package.gni") + +ten_package("openai_chatgpt_python") { + package_kind = "extension" + + resources = [ + "__init__.py", + "addon.py", + "extension.py", + "log.py", + "manifest.json", + "property.json", + ] +} diff --git a/agents/ten_packages/extension/openai_chatgpt_python/README.md b/agents/ten_packages/extension/openai_chatgpt_python/README.md new file mode 100644 index 00000000..2a9b1c82 --- /dev/null +++ b/agents/ten_packages/extension/openai_chatgpt_python/README.md @@ -0,0 +1,60 @@ +# openai_chatgpt_python + +An extension for integrating OpenAI's GPT models (e.g., GPT-4) into your application, providing configurable AI-driven features such as conversational agents, task automation, and tool integration. + +## Features + + + +- OpenAI GPT Integration: Leverage GPT models for text processing and conversational tasks. +- Configurable: Easily customize API keys, model settings, prompts, temperature, etc. +- Async Queue Processing: Supports real-time message processing with task cancellation and prioritization. +- Tool Support: Integrate external tools like image recognition via OpenAI's API. + +## API + +Refer to `api` definition in [manifest.json] and default values in [property.json](property.json). + + + +| **Property** | **Type** | **Description** | +|----------------------------|------------|-------------------------------------------| +| `api_key` | `string` | API key for authenticating with OpenAI | +| `frequency_penalty` | `float64` | Controls how much to penalize new tokens based on their existing frequency in the text so far | +| `presence_penalty` | `float64` | Controls how much to penalize new tokens based on whether they appear in the text so far | +| `temperature` | `float64` | Sampling temperature, higher values mean more randomness | +| `top_p` | `float64` | Nucleus sampling, chooses tokens with cumulative probability `p` | +| `model` | `string` | Model identifier (e.g., GPT-3.5, GPT-4) | +| `max_tokens` | `int64` | Maximum number of tokens to generate | +| `base_url` | `string` | API base URL | +| `prompt` | `string` | Default prompt to send to the model | +| `greeting` | `string` | Greeting message to be used | +| `checking_vision_text_items`| `string` | Items for checking vision-based text responses | +| `proxy_url` | `string` | URL of the proxy server | +| `max_memory_length` | `int64` | Maximum memory length for processing | +| `enable_tools` | `bool` | Flag to enable or disable external tools | + +### Data In: +| **Name** | **Property** | **Type** | **Description** | +|----------------|--------------|------------|-------------------------------| +| `text_data` | `text` | `string` | Incoming text data | + +### Data Out: +| **Name** | **Property** | **Type** | **Description** | +|----------------|--------------|------------|-------------------------------| +| `text_data` | `text` | `string` | Outgoing text data | + +### Command In: +| **Name** | **Description** | +|----------------|---------------------------------------------| +| `flush` | Command to flush the current processing state | + +### Command Out: +| **Name** | **Description** | +|----------------|---------------------------------------------| +| `flush` | Response after flushing the current state | + +### Video Frame In: +| **Name** | **Description** | +|------------------|-------------------------------------------| +| `video_frame` | Video frame input for vision processing | diff --git a/agents/ten_packages/extension/openai_chatgpt_python/__init__.py b/agents/ten_packages/extension/openai_chatgpt_python/__init__.py index 42c4cd12..09a409ff 100644 --- a/agents/ten_packages/extension/openai_chatgpt_python/__init__.py +++ b/agents/ten_packages/extension/openai_chatgpt_python/__init__.py @@ -1,4 +1,11 @@ -from . import openai_chatgpt_addon +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from . import addon from .log import logger logger.info("openai_chatgpt_python extension loaded") diff --git a/agents/ten_packages/extension/openai_chatgpt_python/addon.py b/agents/ten_packages/extension/openai_chatgpt_python/addon.py new file mode 100644 index 00000000..ee13b156 --- /dev/null +++ b/agents/ten_packages/extension/openai_chatgpt_python/addon.py @@ -0,0 +1,22 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from ten import ( + Addon, + register_addon_as_extension, + TenEnv, +) +from .extension import OpenAIChatGPTExtension +from .log import logger + + +@register_addon_as_extension("openai_chatgpt_python") +class OpenAIChatGPTExtensionAddon(Addon): + + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + logger.info("OpenAIChatGPTExtensionAddon on_create_instance") + ten_env.on_create_instance_done(OpenAIChatGPTExtension(name), context) diff --git a/agents/ten_packages/extension/openai_chatgpt_python/extension.py b/agents/ten_packages/extension/openai_chatgpt_python/extension.py new file mode 100644 index 00000000..5d027267 --- /dev/null +++ b/agents/ten_packages/extension/openai_chatgpt_python/extension.py @@ -0,0 +1,318 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +import asyncio +import json +import random +import threading +import traceback + +from .helper import AsyncEventEmitter, AsyncQueue, get_current_time, get_property_bool, get_property_float, get_property_int, get_property_string, parse_sentences, rgb2base64jpeg +from .openai import OpenAIChatGPT, OpenAIChatGPTConfig +from ten import ( + AudioFrame, + VideoFrame, + Extension, + TenEnv, + Cmd, + StatusCode, + CmdResult, + Data, +) +from .log import logger + +CMD_IN_FLUSH = "flush" +CMD_OUT_FLUSH = "flush" +DATA_IN_TEXT_DATA_PROPERTY_TEXT = "text" +DATA_IN_TEXT_DATA_PROPERTY_IS_FINAL = "is_final" +DATA_OUT_TEXT_DATA_PROPERTY_TEXT = "text" +DATA_OUT_TEXT_DATA_PROPERTY_TEXT_END_OF_SEGMENT = "end_of_segment" + +PROPERTY_BASE_URL = "base_url" # Optional +PROPERTY_API_KEY = "api_key" # Required +PROPERTY_MODEL = "model" # Optional +PROPERTY_PROMPT = "prompt" # Optional +PROPERTY_FREQUENCY_PENALTY = "frequency_penalty" # Optional +PROPERTY_PRESENCE_PENALTY = "presence_penalty" # Optional +PROPERTY_TEMPERATURE = "temperature" # Optional +PROPERTY_TOP_P = "top_p" # Optional +PROPERTY_MAX_TOKENS = "max_tokens" # Optional +PROPERTY_GREETING = "greeting" # Optional +PROPERTY_ENABLE_TOOLS = "enable_tools" # Optional +PROPERTY_PROXY_URL = "proxy_url" # Optional +PROPERTY_MAX_MEMORY_LENGTH = "max_memory_length" # Optional +PROPERTY_CHECKING_VISION_TEXT_ITEMS = "checking_vision_text_items" # Optional + + +TASK_TYPE_CHAT_COMPLETION = "chat_completion" +TASK_TYPE_CHAT_COMPLETION_WITH_VISION = "chat_completion_with_vision" + +class OpenAIChatGPTExtension(Extension): + memory = [] + max_memory_length = 10 + openai_chatgpt = None + enable_tools = False + image_data = None + image_width = 0 + image_height = 0 + checking_vision_text_items = [] + loop = None + sentence_fragment = "" + + # Create the queue for message processing + queue = AsyncQueue() + + available_tools = [ + { + "type": "function", + "function": { + # ensure you use gpt-4o or later model if you need image recognition, gpt-4o-mini does not work quite well in this case + "name": "get_vision_image", + "description": "Get the image from camera. Call this whenever you need to understand the input camera image like you have vision capability, for example when user asks 'What can you see?' or 'Can you see me?'", + }, + "strict": True, + } + ] + + def on_init(self, ten_env: TenEnv) -> None: + logger.info("on_init") + ten_env.on_init_done() + + def on_start(self, ten_env: TenEnv) -> None: + logger.info("on_start") + + self.loop = asyncio.new_event_loop() + def start_loop(): + asyncio.set_event_loop(self.loop) + self.loop.run_forever() + threading.Thread(target=start_loop, args=[]).start() + + self.loop.create_task(self._process_queue(ten_env)) + + # Prepare configuration + openai_chatgpt_config = OpenAIChatGPTConfig.default_config() + + # Mandatory properties + openai_chatgpt_config.base_url = get_property_string(ten_env, PROPERTY_BASE_URL) or openai_chatgpt_config.base_url + openai_chatgpt_config.api_key = get_property_string(ten_env, PROPERTY_API_KEY) + if not openai_chatgpt_config.api_key: + logger.info(f"API key is missing, exiting on_start") + return + + # Optional properties + openai_chatgpt_config.model = get_property_string(ten_env, PROPERTY_MODEL) or openai_chatgpt_config.model + openai_chatgpt_config.prompt = get_property_string(ten_env, PROPERTY_PROMPT) or openai_chatgpt_config.prompt + openai_chatgpt_config.frequency_penalty = get_property_float(ten_env, PROPERTY_FREQUENCY_PENALTY) or openai_chatgpt_config.frequency_penalty + openai_chatgpt_config.presence_penalty = get_property_float(ten_env, PROPERTY_PRESENCE_PENALTY) or openai_chatgpt_config.presence_penalty + openai_chatgpt_config.temperature = get_property_float(ten_env, PROPERTY_TEMPERATURE) or openai_chatgpt_config.temperature + openai_chatgpt_config.top_p = get_property_float(ten_env, PROPERTY_TOP_P) or openai_chatgpt_config.top_p + openai_chatgpt_config.max_tokens = get_property_int(ten_env, PROPERTY_MAX_TOKENS) or openai_chatgpt_config.max_tokens + openai_chatgpt_config.proxy_url = get_property_string(ten_env, PROPERTY_PROXY_URL) or openai_chatgpt_config.proxy_url + + # Properties that don't affect openai_chatgpt_config + greeting = get_property_string(ten_env, PROPERTY_GREETING) + self.enable_tools = get_property_bool(ten_env, PROPERTY_ENABLE_TOOLS) + self.max_memory_length = get_property_int(ten_env, PROPERTY_MAX_MEMORY_LENGTH) + checking_vision_text_items_str = get_property_string(ten_env, PROPERTY_CHECKING_VISION_TEXT_ITEMS) + if checking_vision_text_items_str: + try: + self.checking_vision_text_items = json.loads(checking_vision_text_items_str) + except Exception as err: + logger.info(f"Error parsing {PROPERTY_CHECKING_VISION_TEXT_ITEMS}: {err}") + + # Create instance + try: + self.openai_chatgpt = OpenAIChatGPT(openai_chatgpt_config) + logger.info(f"initialized with max_tokens: {openai_chatgpt_config.max_tokens}, model: {openai_chatgpt_config.model}") + except Exception as err: + logger.info(f"Failed to initialize OpenAIChatGPT: {err}") + + # Send greeting if available + if greeting: + try: + output_data = Data.create("text_data") + output_data.set_property_string(DATA_OUT_TEXT_DATA_PROPERTY_TEXT, greeting) + output_data.set_property_bool(DATA_OUT_TEXT_DATA_PROPERTY_TEXT_END_OF_SEGMENT, True) + ten_env.send_data(output_data) + logger.info(f"Greeting [{greeting}] sent") + except Exception as err: + logger.info(f"Failed to send greeting [{greeting}]: {err}") + ten_env.on_start_done() + + def on_stop(self, ten_env: TenEnv) -> None: + logger.info("on_stop") + + # TODO: clean up resources + + ten_env.on_stop_done() + + def on_deinit(self, ten_env: TenEnv) -> None: + logger.info("on_deinit") + ten_env.on_deinit_done() + + def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: + logger.info(f"on_cmd json: {cmd.to_json()}") + + cmd_name = cmd.get_name() + + if cmd_name == CMD_IN_FLUSH: + asyncio.run_coroutine_threadsafe(self._flush_queue(), self.loop) + ten_env.send_cmd(Cmd.create(CMD_OUT_FLUSH), None) + logger.info("on_cmd sent flush") + status_code, detail = StatusCode.OK, "success" + else: + logger.info(f"on_cmd unknown cmd: {cmd_name}") + status_code, detail = StatusCode.ERROR, "unknown cmd" + + cmd_result = CmdResult.create(status_code) + cmd_result.set_property_string("detail", detail) + ten_env.return_result(cmd_result, cmd) + + def on_data(self, ten_env: TenEnv, data: Data) -> None: + # Get the necessary properties + is_final = get_property_bool(data, DATA_IN_TEXT_DATA_PROPERTY_IS_FINAL) + input_text = get_property_string(data, DATA_IN_TEXT_DATA_PROPERTY_TEXT) + + if not is_final: + logger.info("ignore non-final input") + return + if not input_text: + logger.info("ignore empty text") + return + + logger.info(f"OnData input text: [{input_text}]") + + # Start an asynchronous task for handling chat completion + asyncio.run_coroutine_threadsafe(self.queue.put([TASK_TYPE_CHAT_COMPLETION, input_text]), self.loop) + + def on_audio_frame(self, ten_env: TenEnv, audio_frame: AudioFrame) -> None: + # TODO: process pcm frame + pass + + def on_video_frame(self, ten_env: TenEnv, video_frame: VideoFrame) -> None: + # logger.info(f"OpenAIChatGPTExtension on_video_frame {frame.get_width()} {frame.get_height()}") + self.image_data = video_frame.get_buf() + self.image_width = video_frame.get_width() + self.image_height = video_frame.get_height() + return + + async def _process_queue(self, ten_env: TenEnv): + """Asynchronously process queue items one by one.""" + while True: + # Wait for an item to be available in the queue + [task_type, message] = await self.queue.get() + try: + # Create a new task for the new message + self.current_task = asyncio.create_task(self._run_chatflow(ten_env, task_type, message, self.memory)) + await self.current_task # Wait for the current task to finish or be cancelled + except asyncio.CancelledError: + logger.info(f"Task cancelled: {message}") + + async def _flush_queue(self): + """Flushes the self.queue and cancels the current task.""" + # Flush the queue using the new flush method + await self.queue.flush() + + # Cancel the current task if one is running + if self.current_task: + logger.info("Cancelling the current task during flush.") + self.current_task.cancel() + + async def _run_chatflow(self, ten_env: TenEnv, task_type:str, input_text: str, memory): + """Run the chatflow asynchronously.""" + memory_cache = [] + try: + logger.info(f"for input text: [{input_text}] memory: {memory}") + message = None + tools = None + + # Prepare the message and tools based on the task type + if task_type == TASK_TYPE_CHAT_COMPLETION: + message = {"role": "user", "content": input_text} + memory_cache = memory_cache + [message, {"role": "assistant", "content": ""}] + tools = self.available_tools if self.enable_tools else None + elif task_type == TASK_TYPE_CHAT_COMPLETION_WITH_VISION: + message = {"role": "user", "content": input_text} + memory_cache = memory_cache + [message, {"role": "assistant", "content": ""}] + tools = self.available_tools if self.enable_tools else None + if self.image_data is not None: + url = rgb2base64jpeg(self.image_data, self.image_width, self.image_height) + message = { + "role": "user", + "content": [ + {"type": "text", "text": input_text}, + {"type": "image_url", "image_url": {"url": url}}, + ], + } + logger.info(f"msg with vision data: {message}") + + + self.sentence_fragment = "" + + # Create an asyncio.Event to signal when content is finished + content_finished_event = asyncio.Event() + + # Create an async listener to handle tool calls and content updates + async def handle_tool_call(tool_call): + logger.info(f"tool_call: {tool_call}") + if tool_call.function.name == "get_vision_image": + # Append the vision image to the last assistant message + await self.queue.put([TASK_TYPE_CHAT_COMPLETION_WITH_VISION, input_text], True) + + async def handle_content_update(content:str): + # Append the content to the last assistant message + for item in reversed(memory_cache): + if item.get('role') == 'assistant': + item['content'] = item['content'] + content + break + sentences, self.sentence_fragment = parse_sentences(self.sentence_fragment, content) + for s in sentences: + self._send_data(ten_env, s, False) + + async def handle_content_finished(full_content:str): + content_finished_event.set() + + listener = AsyncEventEmitter() + listener.on("tool_call", handle_tool_call) + listener.on("content_update", handle_content_update) + listener.on("content_finished", handle_content_finished) + + # Make an async API call to get chat completions + await self.openai_chatgpt.get_chat_completions_stream(memory + [message], tools, listener) + + # Wait for the content to be finished + await content_finished_event.wait() + except asyncio.CancelledError: + logger.info(f"Task cancelled: {input_text}") + except Exception as e: + logger.error(f"Error in chat_completion: {traceback.format_exc()} for input text: {input_text}") + finally: + self._send_data(ten_env, "", True) + # always append the memory + for m in memory_cache: + self._append_memory(m) + + def _append_memory(self, message:str): + if len(self.memory) > self.max_memory_length: + self.memory.pop(0) + self.memory.append(message) + + def _send_data(self, ten_env: TenEnv, sentence: str, end_of_segment: bool): + try: + output_data = Data.create("text_data") + output_data.set_property_string(DATA_OUT_TEXT_DATA_PROPERTY_TEXT, sentence) + output_data.set_property_bool( + DATA_OUT_TEXT_DATA_PROPERTY_TEXT_END_OF_SEGMENT, end_of_segment + ) + ten_env.send_data(output_data) + logger.info( + f"{'end of segment ' if end_of_segment else ''}sent sentence [{sentence}]" + ) + except Exception as err: + logger.info( + f"send sentence [{sentence}] failed, err: {err}" + ) \ No newline at end of file diff --git a/agents/ten_packages/extension/openai_chatgpt_python/helper.py b/agents/ten_packages/extension/openai_chatgpt_python/helper.py new file mode 100644 index 00000000..28c28f19 --- /dev/null +++ b/agents/ten_packages/extension/openai_chatgpt_python/helper.py @@ -0,0 +1,187 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +import asyncio +from collections import deque +from ten.data import Data +from .log import logger +from PIL import Image +from datetime import datetime +from io import BytesIO +from base64 import b64encode + + +def get_property_bool(data: Data, property_name: str) -> bool: + """Helper to get boolean property from data with error handling.""" + try: + return data.get_property_bool(property_name) + except Exception as err: + logger.warn(f"GetProperty {property_name} failed: {err}") + return False + +def get_property_string(data: Data, property_name: str) -> str: + """Helper to get string property from data with error handling.""" + try: + return data.get_property_string(property_name) + except Exception as err: + logger.warn(f"GetProperty {property_name} failed: {err}") + return "" + +def get_property_int(data: Data, property_name: str) -> int: + """Helper to get int property from data with error handling.""" + try: + return data.get_property_int(property_name) + except Exception as err: + logger.warn(f"GetProperty {property_name} failed: {err}") + return 0 + +def get_property_float(data: Data, property_name: str) -> float: + """Helper to get float property from data with error handling.""" + try: + return data.get_property_float(property_name) + except Exception as err: + logger.warn(f"GetProperty {property_name} failed: {err}") + return 0.0 + + +def get_current_time(): + # Get the current time + start_time = datetime.now() + # Get the number of microseconds since the Unix epoch + unix_microseconds = int(start_time.timestamp() * 1_000_000) + return unix_microseconds + + +def is_punctuation(char): + if char in [",", "，", ".", "。", "?", "？", "!", "！"]: + return True + return False + + +def parse_sentences(sentence_fragment, content): + sentences = [] + current_sentence = sentence_fragment + for char in content: + current_sentence += char + if is_punctuation(char): + # Check if the current sentence contains non-punctuation characters + stripped_sentence = current_sentence + if any(c.isalnum() for c in stripped_sentence): + sentences.append(stripped_sentence) + current_sentence = "" # Reset for the next sentence + + remain = current_sentence # Any remaining characters form the incomplete sentence + return sentences, remain + + + +def rgb2base64jpeg(rgb_data, width, height): + # Convert the RGB image to a PIL Image + pil_image = Image.frombytes("RGBA", (width, height), bytes(rgb_data)) + pil_image = pil_image.convert("RGB") + + # Resize the image while maintaining its aspect ratio + pil_image = resize_image_keep_aspect(pil_image, 320) + + # Save the image to a BytesIO object in JPEG format + buffered = BytesIO() + pil_image.save(buffered, format="JPEG") + # pil_image.save("test.jpg", format="JPEG") + + # Get the byte data of the JPEG image + jpeg_image_data = buffered.getvalue() + + # Convert the JPEG byte data to a Base64 encoded string + base64_encoded_image = b64encode(jpeg_image_data).decode("utf-8") + + # Create the data URL + mime_type = "image/jpeg" + base64_url = f"data:{mime_type};base64,{base64_encoded_image}" + return base64_url + + +def resize_image_keep_aspect(image, max_size=512): + """ + Resize an image while maintaining its aspect ratio, ensuring the larger dimension is max_size. + If both dimensions are smaller than max_size, the image is not resized. + + :param image: A PIL Image object + :param max_size: The maximum size for the larger dimension (width or height) + :return: A PIL Image object (resized or original) + """ + # Get current width and height + width, height = image.size + + # If both dimensions are already smaller than max_size, return the original image + if width <= max_size and height <= max_size: + return image + + # Calculate the aspect ratio + aspect_ratio = width / height + + # Determine the new dimensions + if width > height: + new_width = max_size + new_height = int(max_size / aspect_ratio) + else: + new_height = max_size + new_width = int(max_size * aspect_ratio) + + # Resize the image with the new dimensions + resized_image = image.resize((new_width, new_height)) + + return resized_image + + +class AsyncEventEmitter: + def __init__(self): + self.listeners = {} + + def on(self, event_name, listener): + """Register an event listener.""" + if event_name not in self.listeners: + self.listeners[event_name] = [] + self.listeners[event_name].append(listener) + + def emit(self, event_name, *args, **kwargs): + """Fire the event without waiting for listeners to finish.""" + if event_name in self.listeners: + for listener in self.listeners[event_name]: + asyncio.create_task(listener(*args, **kwargs)) + + +class AsyncQueue: + def __init__(self): + self._queue = deque() # Use deque for efficient prepend and append + self._condition = asyncio.Condition() # Use Condition to manage access + + async def put(self, item, prepend=False): + """Add an item to the queue (prepend if specified).""" + async with self._condition: + if prepend: + self._queue.appendleft(item) # Prepend item to the front + else: + self._queue.append(item) # Append item to the back + self._condition.notify() + + async def get(self): + """Remove and return an item from the queue.""" + async with self._condition: + while not self._queue: + await self._condition.wait() # Wait until an item is available + return self._queue.popleft() # Pop from the front of the deque + + async def flush(self): + """Flush all items from the queue.""" + async with self._condition: + while self._queue: + self._queue.popleft() # Clear the queue + self._condition.notify_all() # Notify all consumers that the queue is empty + + def __len__(self): + """Return the current size of the queue.""" + return len(self._queue) \ No newline at end of file diff --git a/agents/ten_packages/extension/openai_chatgpt_python/log.py b/agents/ten_packages/extension/openai_chatgpt_python/log.py index fa2202da..1813e965 100644 --- a/agents/ten_packages/extension/openai_chatgpt_python/log.py +++ b/agents/ten_packages/extension/openai_chatgpt_python/log.py @@ -1,11 +1,20 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# import logging logger = logging.getLogger("openai_chatgpt_python") logger.setLevel(logging.INFO) -formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(process)d - [%(filename)s:%(lineno)d] - %(message)s" +formatter_str = ( + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - " + "[%(filename)s:%(lineno)d] - %(message)s" ) +formatter = logging.Formatter(formatter_str) console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) diff --git a/agents/ten_packages/extension/openai_chatgpt_python/manifest.json b/agents/ten_packages/extension/openai_chatgpt_python/manifest.json index b23cb9d8..4a74e306 100644 --- a/agents/ten_packages/extension/openai_chatgpt_python/manifest.json +++ b/agents/ten_packages/extension/openai_chatgpt_python/manifest.json @@ -1,14 +1,24 @@ { "type": "extension", "name": "openai_chatgpt_python", - "version": "0.4.0", + "version": "0.1.0", "dependencies": [ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], + "package": { + "include": [ + "manifest.json", + "property.json", + "BUILD.gn", + "**.tent", + "**.py", + "README.md" + ] + }, "api": { "property": { "api_key": { diff --git a/agents/ten_packages/extension/openai_chatgpt_python/openai.py b/agents/ten_packages/extension/openai_chatgpt_python/openai.py new file mode 100644 index 00000000..3449126d --- /dev/null +++ b/agents/ten_packages/extension/openai_chatgpt_python/openai.py @@ -0,0 +1,125 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +import random +import requests +from openai import AsyncOpenAI +from typing import List, Dict, Any, Optional +from .log import logger + + +class OpenAIChatGPTConfig: + def __init__(self, + base_url: str, + api_key: str, + model: str, + prompt: str, + frequency_penalty: float, + presence_penalty: float, + top_p: float, + temperature: float, + max_tokens: int, + seed: Optional[int] = None, + proxy_url: Optional[str] = None): + self.base_url = base_url + self.api_key = api_key + self.model = model + self.prompt = prompt + self.frequency_penalty = frequency_penalty + self.presence_penalty = presence_penalty + self.top_p = top_p + self.temperature = temperature + self.max_tokens = max_tokens + self.seed = seed if seed is not None else random.randint(0, 10000) + self.proxy_url = proxy_url + + @classmethod + def default_config(cls): + return cls( + base_url="https://api.openai.com/v1", + api_key="", + model="gpt-4", # Adjust this to match the equivalent of `openai.GPT4o` in the Python library + prompt="You are a voice assistant who talks in a conversational way and can chat with me like my friends. I will speak to you in English or Chinese, and you will answer in the corrected and improved version of my text with the language I use. Don’t talk like a robot, instead I would like you to talk like a real human with emotions. I will use your answer for text-to-speech, so don’t return me any meaningless characters. I want you to be helpful, when I’m asking you for advice, give me precise, practical and useful advice instead of being vague. When giving me a list of options, express the options in a narrative way instead of bullet points.", + frequency_penalty=0.9, + presence_penalty=0.9, + top_p=1.0, + temperature=0.1, + max_tokens=512, + seed=random.randint(0, 10000), + proxy_url="" + ) + + +class OpenAIChatGPT: + client = None + def __init__(self, config: OpenAIChatGPTConfig): + self.config = config + logger.info(f"OpenAIChatGPT initialized with config: {config.api_key}") + self.client = AsyncOpenAI( + api_key=config.api_key, + base_url=config.base_url + ) + self.session = requests.Session() + if config.proxy_url: + proxies = { + "http": config.proxy_url, + "https": config.proxy_url, + } + self.session.proxies.update(proxies) + self.client.session = self.session + + async def get_chat_completions_stream(self, messages, tools = None, listener = None): + req = { + "model": self.config.model, + "messages": [ + { + "role": "system", + "content": self.config.prompt, + }, + *messages, + ], + "tools": tools, + "temperature": self.config.temperature, + "top_p": self.config.top_p, + "presence_penalty": self.config.presence_penalty, + "frequency_penalty": self.config.frequency_penalty, + "max_tokens": self.config.max_tokens, + "seed": self.config.seed, + "stream": True, + } + + try: + response = await self.client.chat.completions.create(**req) + except Exception as e: + raise Exception(f"CreateChatCompletionStream failed, err: {e}") + + full_content = "" + + async for chat_completion in response: + choice = chat_completion.choices[0] + delta = choice.delta + + content = delta.content if delta and delta.content else "" + + # Emit content update event (fire-and-forget) + if listener and content: + listener.emit('content_update', content) + + full_content += content + + # Check for tool calls + if delta.tool_calls: + for tool_call in delta.tool_calls: + logger.info(f"tool_call: {tool_call}") + + # Emit tool call event (fire-and-forget) + if listener: + listener.emit('tool_call', tool_call) + + # Emit content finished event after the loop completes + if listener: + listener.emit('content_finished', full_content) diff --git a/agents/ten_packages/extension/openai_chatgpt_python/requirements.txt b/agents/ten_packages/extension/openai_chatgpt_python/requirements.txt index ca4978c3..51cdd053 100644 --- a/agents/ten_packages/extension/openai_chatgpt_python/requirements.txt +++ b/agents/ten_packages/extension/openai_chatgpt_python/requirements.txt @@ -1,4 +1,4 @@ openai numpy -requests==2.32.3 -pillow==10.4.0 \ No newline at end of file +requests +pillow \ No newline at end of file diff --git a/agents/ten_packages/extension/polly_tts/manifest.json b/agents/ten_packages/extension/polly_tts/manifest.json index dfbc94ea..932c3b47 100644 --- a/agents/ten_packages/extension/polly_tts/manifest.json +++ b/agents/ten_packages/extension/polly_tts/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/qwen_llm_python/manifest.json b/agents/ten_packages/extension/qwen_llm_python/manifest.json index a05290af..bdad7d87 100644 --- a/agents/ten_packages/extension/qwen_llm_python/manifest.json +++ b/agents/ten_packages/extension/qwen_llm_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/agents/ten_packages/extension/transcribe_asr_python/manifest.json b/agents/ten_packages/extension/transcribe_asr_python/manifest.json index 69386329..5950e3f7 100644 --- a/agents/ten_packages/extension/transcribe_asr_python/manifest.json +++ b/agents/ten_packages/extension/transcribe_asr_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1" + "version": "0.2" } ], "api": { diff --git a/demo/.dockerignore b/demo/.dockerignore new file mode 100644 index 00000000..80ae13ce --- /dev/null +++ b/demo/.dockerignore @@ -0,0 +1,3 @@ +.git +.next +node_modules diff --git a/demo/.env b/demo/.env new file mode 100644 index 00000000..5f92b324 --- /dev/null +++ b/demo/.env @@ -0,0 +1 @@ +AGENT_SERVER_URL=http://localhost:8080 \ No newline at end of file diff --git a/demo/.gitignore b/demo/.gitignore new file mode 100644 index 00000000..3bcf2bf5 --- /dev/null +++ b/demo/.gitignore @@ -0,0 +1,135 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +# .env +!.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# lock +package-lock.json +yarn.lock diff --git a/demo/Dockerfile b/demo/Dockerfile new file mode 100644 index 00000000..63379d47 --- /dev/null +++ b/demo/Dockerfile @@ -0,0 +1,27 @@ +FROM node:20-alpine AS base + +FROM base AS builder + +WORKDIR /app + +# COPY .env.example .env +COPY . . + +RUN npm i --verbose && \ + npm run build + + +FROM base AS runner + +WORKDIR /app + +ENV NODE_ENV production + +RUN mkdir .next + +COPY --from=builder /app/.next/standalone ./ +COPY --from=builder /app/.next/static ./.next/static + +EXPOSE 3000 + +CMD HOSTNAME="0.0.0.0" node server.js \ No newline at end of file diff --git a/demo/LICENSE b/demo/LICENSE new file mode 100644 index 00000000..e4589a2b --- /dev/null +++ b/demo/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Agora Community + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/demo/next-env.d.ts b/demo/next-env.d.ts new file mode 100644 index 00000000..4f11a03d --- /dev/null +++ b/demo/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/basic-features/typescript for more information. diff --git a/demo/next.config.mjs b/demo/next.config.mjs new file mode 100644 index 00000000..132f5fc1 --- /dev/null +++ b/demo/next.config.mjs @@ -0,0 +1,37 @@ +/** @type {import('next').NextConfig} */ + +const nextConfig = { + // basePath: '/ai-agent', + // output: 'export', + output: 'standalone', + reactStrictMode: false, + webpack(config) { + // Grab the existing rule that handles SVG imports + const fileLoaderRule = config.module.rules.find((rule) => + rule.test?.test?.('.svg'), + ) + + config.module.rules.push( + // Reapply the existing rule, but only for svg imports ending in ?url + { + ...fileLoaderRule, + test: /\.svg$/i, + resourceQuery: /url/, // *.svg?url + }, + // Convert all other *.svg imports to React components + { + test: /\.svg$/i, + issuer: fileLoaderRule.issuer, + resourceQuery: { not: [...fileLoaderRule.resourceQuery.not, /url/] }, // exclude if *.svg?url + use: ['@svgr/webpack'], + }, + ) + + // Modify the file loader rule to ignore *.svg, since we have it handled now. + fileLoaderRule.exclude = /\.svg$/i + + return config + } +}; + +export default nextConfig; diff --git a/demo/package.json b/demo/package.json new file mode 100644 index 00000000..668d2d3f --- /dev/null +++ b/demo/package.json @@ -0,0 +1,42 @@ +{ + "name": "astra.ai-playground", + "version": "0.4.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint", + "proto": "pbjs -t json-module -w commonjs -o src/protobuf/SttMessage.js src/protobuf/SttMessage.proto" + }, + "dependencies": { + "@ant-design/icons": "^5.3.7", + "@reduxjs/toolkit": "^2.2.3", + "agora-rtc-sdk-ng": "^4.21.0", + "antd": "^5.15.3", + "axios": "^1.7.7", + "next": "14.2.4", + "protobufjs": "^7.2.5", + "react": "^18", + "react-colorful": "^5.6.1", + "react-dom": "^18", + "react-redux": "^9.1.0", + "redux": "^5.0.1" + }, + "devDependencies": { + "@minko-fe/postcss-pxtoviewport": "^1.3.2", + "@svgr/webpack": "^8.1.0", + "@types/node": "^20", + "@types/react": "^18", + "@types/react-dom": "^18", + "@types/react-redux": "^7.1.22", + "autoprefixer": "^10.4.16", + "eslint": "^8", + "eslint-config-next": "14.2.4", + "postcss": "^8.4.31", + "protobufjs-cli": "^1.1.2", + "sass": "^1.77.5", + "typescript": "^5" + }, + "packageManager": "yarn@1.22.22+sha1.ac34549e6aa8e7ead463a7407e1c7390f61a6610" +} diff --git a/demo/postcss.config.js b/demo/postcss.config.js new file mode 100644 index 00000000..ea748d4d --- /dev/null +++ b/demo/postcss.config.js @@ -0,0 +1,10 @@ +module.exports = { + plugins: { + autoprefixer: {}, + "@minko-fe/postcss-pxtoviewport": { + viewportWidth: 375, + exclude: /node_modules/, + include: /\/src\/platform\/mobile\//, + } + }, +} diff --git a/demo/src/app/api/agents/start/graph.tsx b/demo/src/app/api/agents/start/graph.tsx new file mode 100644 index 00000000..cc43137e --- /dev/null +++ b/demo/src/app/api/agents/start/graph.tsx @@ -0,0 +1,159 @@ +import { LanguageMap } from '@/common/constant'; +import { NextRequest, NextResponse } from 'next/server'; + + +const { AGENT_SERVER_URL } = process.env; + +// Check if environment variables are available +if (!AGENT_SERVER_URL) { + throw "Environment variables AGENT_SERVER_URL are not available"; +} + + +export const voiceNameMap: LanguageMap = { + "zh-CN": { + azure: { + male: "zh-CN-YunxiNeural", + female: "zh-CN-XiaoxiaoNeural", + }, + elevenlabs: { + male: "pNInz6obpgDQGcFmaJgB", // Adam + female: "Xb7hH8MSUJpSbSDYk0k2", // Alice + }, + polly: { + male: "Zhiyu", + female: "Zhiyu", + }, + }, + "en-US": { + azure: { + male: "en-US-BrianNeural", + female: "en-US-JaneNeural", + }, + elevenlabs: { + male: "pNInz6obpgDQGcFmaJgB", // Adam + female: "Xb7hH8MSUJpSbSDYk0k2", // Alice + }, + polly: { + male: "Matthew", + female: "Ruth", + }, + }, + "ja-JP": { + azure: { + male: "ja-JP-KeitaNeural", + female: "ja-JP-NanamiNeural", + }, + }, + "ko-KR": { + azure: { + male: "ko-KR-InJoonNeural", + female: "ko-KR-JiMinNeural", + }, + }, +}; + +// Get the graph properties based on the graph name, language, and voice type +// This is the place where you can customize the properties for different graphs to override default property.json +export const getGraphProperties = (graphName: string, language: string, voiceType: string) => { + let localizationOptions = { + "greeting": "TEN agent connected. How can I help you today?", + "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", + } + + if (language === "zh-CN") { + localizationOptions = { + "greeting": "TEN Agent 已连接，需要我为您提供什么帮助?", + "checking_vision_text_items": "[\"让我看看你的摄像头...\",\"让我看一下...\",\"我看一下，请稍候...\"]", + } + } else if (language === "ja-JP") { + localizationOptions = { + "greeting": "TEN Agent エージェントに接続されました。今日は何をお手伝いしましょうか?", + "checking_vision_text_items": "[\"ちょっと見てみます...\",\"カメラをチェックします...\",\"少々お待ちください...\"]", + } + } else if (language === "ko-KR") { + localizationOptions = { + "greeting": "TEN Agent 에이전트에 연결되었습니다. 오늘은 무엇을 도와드릴까요?", + "checking_vision_text_items": "[\"조금만 기다려 주세요...\",\"카메라를 확인해 보겠습니다...\",\"잠시만 기다려 주세요...\"]", + } + } + + if (graphName == "camera.va.openai.azure") { + return { + "agora_rtc": { + "agora_asr_language": language, + }, + "openai_chatgpt": { + "model": "gpt-4o", + ...localizationOptions + }, + "azure_tts": { + "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] + } + } + } else if (graphName == "va.openai.azure") { + return { + "agora_rtc": { + "agora_asr_language": language, + }, + "openai_chatgpt": { + "model": "gpt-4o-mini", + ...localizationOptions + }, + "azure_tts": { + "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] + } + } + } else if (graphName == "va.qwen.rag") { + return { + "agora_rtc": { + "agora_asr_language": language, + }, + "azure_tts": { + "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] + } + } + } + return {} +} + +export async function startAgent(request: NextRequest) { + try{ + const body = await request.json(); + const { + request_id, + channel_name, + user_uid, + graph_name, + language, + voice_type, + } = body; + + // Send a POST request to start the agent + const response = await fetch(`${AGENT_SERVER_URL}/start`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + request_id, + channel_name, + user_uid, + graph_name, + // Get the graph properties based on the graph name, language, and voice type + properties: getGraphProperties(graph_name, language, voice_type), + }), + }); + + const responseData = await response.json(); + + return NextResponse.json(responseData, { status: response.status }); + } catch (error) { + if (error instanceof Response) { + const errorData = await error.json(); + return NextResponse.json(errorData, { status: error.status }); + } else { + return NextResponse.json({ code: "1", data: null, msg: "Internal Server Error" }, { status: 500 }); + } + } +} diff --git a/demo/src/app/api/agents/start/route.tsx b/demo/src/app/api/agents/start/route.tsx new file mode 100644 index 00000000..187621a6 --- /dev/null +++ b/demo/src/app/api/agents/start/route.tsx @@ -0,0 +1,61 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getGraphProperties } from './graph'; +import axios from 'axios'; +/** + * Handles the POST request to start an agent. + * + * @param request - The NextRequest object representing the incoming request. + * @returns A NextResponse object representing the response to be sent back to the client. + */ +export async function POST(request: NextRequest) { + try { + const { AGENT_SERVER_URL } = process.env; + + // Check if environment variables are available + if (!AGENT_SERVER_URL) { + throw "Environment variables are not available"; + } + + const body = await request.json(); + const { + request_id, + channel_name, + user_uid, + graph_name, + language, + voice_type, + } = body; + + console.log(`Starting agent for request ID: ${JSON.stringify({ + request_id, + channel_name, + user_uid, + graph_name, + // Get the graph properties based on the graph name, language, and voice type + properties: getGraphProperties(graph_name, language, voice_type), + })}`); + + console.log(`AGENT_SERVER_URL: ${AGENT_SERVER_URL}/start`); + + // Send a POST request to start the agent + const response = await axios.post(`${AGENT_SERVER_URL}/start`, { + request_id, + channel_name, + user_uid, + graph_name, + // Get the graph properties based on the graph name, language, and voice type + properties: getGraphProperties(graph_name, language, voice_type), + }); + + const responseData = response.data; + + return NextResponse.json(responseData, { status: response.status }); + } catch (error) { + if (error instanceof Response) { + const errorData = await error.json(); + return NextResponse.json(errorData, { status: error.status }); + } else { + return NextResponse.json({ code: "1", data: null, msg: "Internal Server Error" }, { status: 500 }); + } + } +} \ No newline at end of file diff --git a/demo/src/app/favicon.ico b/demo/src/app/favicon.ico new file mode 100644 index 00000000..21b38b96 Binary files /dev/null and b/demo/src/app/favicon.ico differ diff --git a/demo/src/app/global.css b/demo/src/app/global.css new file mode 100644 index 00000000..7a1e1861 --- /dev/null +++ b/demo/src/app/global.css @@ -0,0 +1,67 @@ +* { + box-sizing: border-box; + padding: 0; + margin: 0; +} + +html, +body { + background-color: #0F0F11; + font-family: "PingFang SC"; + height: 100%; +} + +a { + color: inherit; + text-decoration: none; +} + +@media (prefers-color-scheme: dark) { + html { + color-scheme: dark; + } +} + +.ant-select-arrow { + color: #667085 !important; +} + + +.ant-select-selection-item { + color: #667085 !important; +} + +.ant-select-selector { + border: 1px solid #272A2F !important; + background-color: #272A2F !important; +} + +.ant-select-dropdown { + background-color: #1E2024 !important; +} + +.ant-select-item { + background: #1E2024 !important; + color: var(--Grey-600, #667085) !important; +} + +.ant-select-item-option-selected { + background: #272A2F !important; + color: var(--Grey-300, #EAECF0) !important; +} + + +.ant-popover-inner { + /* width: 260px !important; */ + background: #1E2025 !important; +} + + +.ant-select-selection-placeholder { + color: var(--Grey-600, #667085) !important; +} + + +.ant-empty-description { + color: var(--Grey-600, #667085) !important; +} diff --git a/playground/src/app/home/page.tsx b/demo/src/app/home/page.tsx similarity index 100% rename from playground/src/app/home/page.tsx rename to demo/src/app/home/page.tsx diff --git a/demo/src/app/index.module.scss b/demo/src/app/index.module.scss new file mode 100644 index 00000000..78585596 --- /dev/null +++ b/demo/src/app/index.module.scss @@ -0,0 +1,93 @@ +@function multiple-box-shadow($n) { + $value: '#{random(2000)}px #{random(2000)}px #FFF'; + + @for $i from 2 through $n { + $value: '#{$value}, #{random(2000)}px #{random(2000)}px #FFF'; + } + + @return unquote($value); +} + +$shadows-small: multiple-box-shadow(700); +$shadows-medium: multiple-box-shadow(200); +$shadows-big: multiple-box-shadow(100); + + +.login { + position: absolute; + left: 0; + top: 0; + width: 100%; + height: 100%; + overflow: hidden; + // background: radial-gradient(ellipse at bottom, #1B2735 0%, #090A0F 100%); + background: url("../assets/background.jpg") no-repeat center center; + background-size: cover; + box-sizing: border-box; + + .starts { + width: 1px; + height: 1px; + background: transparent; + box-shadow: $shadows-small; + animation: animStar 50s linear infinite; + + &:after { + content: " "; + position: absolute; + top: 2000px; + width: 1px; + height: 1px; + background: transparent; + box-shadow: $shadows-small + } + } + + .starts2 { + width: 2px; + height: 2px; + box-shadow: $shadows-medium; + animation: animStar 100s linear infinite; + + &:after { + content: " "; + position: absolute; + top: 2000px; + width: 2px; + height: 2px; + background: transparent; + box-shadow: $shadows-medium; + } + } + + .starts3 { + width: 3px; + height: 3px; + background: transparent; + box-shadow: $shadows-big; + animation: animStar 150s linear infinite; + + &:after { + content: " "; + position: absolute; + top: 2000px; + width: 3px; + height: 3px; + background: transparent; + box-shadow: $shadows-big; + } + + } + +} + + +@keyframes animStar { + from { + transform: translateY(0px) + } + + to { + transform: translateY(-2000px) + } +} diff --git a/demo/src/app/layout.tsx b/demo/src/app/layout.tsx new file mode 100644 index 00000000..b6153573 --- /dev/null +++ b/demo/src/app/layout.tsx @@ -0,0 +1,53 @@ +import { ConfigProvider } from "antd" +import { StoreProvider } from "@/store"; +import type { Metadata, Viewport } from "next"; + +import './global.css' + + +export const metadata: Metadata = { + title: "Astra.ai", + description: "A multimodal agent powered by TEN", + appleWebApp: { + capable: true, + statusBarStyle: "black", + } +}; + +export const viewport: Viewport = { + width: "device-width", + initialScale: 1, + minimumScale: 1, + maximumScale: 1, + userScalable: false, + viewportFit: "cover", +} + + + + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + + + {children} + + + + + ); +} diff --git a/demo/src/app/page.tsx b/demo/src/app/page.tsx new file mode 100644 index 00000000..1bdcdeda --- /dev/null +++ b/demo/src/app/page.tsx @@ -0,0 +1,14 @@ +import LoginCard from "@/components/loginCard" +import styles from "./index.module.scss" + +export default function Login() { + + return ( +
+
+
+
+ +
+ ); +} diff --git a/demo/src/assets/background.jpg b/demo/src/assets/background.jpg new file mode 100644 index 00000000..02762343 Binary files /dev/null and b/demo/src/assets/background.jpg differ diff --git a/demo/src/assets/cam_mute.svg b/demo/src/assets/cam_mute.svg new file mode 100644 index 00000000..a4640ae4 --- /dev/null +++ b/demo/src/assets/cam_mute.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/cam_unmute.svg b/demo/src/assets/cam_unmute.svg new file mode 100644 index 00000000..1eebfaa6 --- /dev/null +++ b/demo/src/assets/cam_unmute.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/color_picker.svg b/demo/src/assets/color_picker.svg new file mode 100644 index 00000000..fb9bb33e --- /dev/null +++ b/demo/src/assets/color_picker.svg @@ -0,0 +1,17 @@ + + + + + + + + + + diff --git a/demo/src/assets/github.svg b/demo/src/assets/github.svg new file mode 100644 index 00000000..e6566c41 --- /dev/null +++ b/demo/src/assets/github.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/info.svg b/demo/src/assets/info.svg new file mode 100644 index 00000000..8ca99511 --- /dev/null +++ b/demo/src/assets/info.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/logo.svg b/demo/src/assets/logo.svg new file mode 100644 index 00000000..af99893a --- /dev/null +++ b/demo/src/assets/logo.svg @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demo/src/assets/logo_small.svg b/demo/src/assets/logo_small.svg new file mode 100644 index 00000000..34e755bd --- /dev/null +++ b/demo/src/assets/logo_small.svg @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demo/src/assets/mic_mute.svg b/demo/src/assets/mic_mute.svg new file mode 100644 index 00000000..dd4a17dd --- /dev/null +++ b/demo/src/assets/mic_mute.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/mic_unmute.svg b/demo/src/assets/mic_unmute.svg new file mode 100644 index 00000000..18e78236 --- /dev/null +++ b/demo/src/assets/mic_unmute.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/network/average.svg b/demo/src/assets/network/average.svg new file mode 100644 index 00000000..9a27072f --- /dev/null +++ b/demo/src/assets/network/average.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/demo/src/assets/network/disconnected.svg b/demo/src/assets/network/disconnected.svg new file mode 100644 index 00000000..b7db1d71 --- /dev/null +++ b/demo/src/assets/network/disconnected.svg @@ -0,0 +1,9 @@ + + + + + diff --git a/demo/src/assets/network/excellent.svg b/demo/src/assets/network/excellent.svg new file mode 100644 index 00000000..55b9fc9e --- /dev/null +++ b/demo/src/assets/network/excellent.svg @@ -0,0 +1,6 @@ + + + + diff --git a/demo/src/assets/network/good.svg b/demo/src/assets/network/good.svg new file mode 100644 index 00000000..8c36a7e7 --- /dev/null +++ b/demo/src/assets/network/good.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/demo/src/assets/network/poor.svg b/demo/src/assets/network/poor.svg new file mode 100644 index 00000000..d9df0238 --- /dev/null +++ b/demo/src/assets/network/poor.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/demo/src/assets/pdf.svg b/demo/src/assets/pdf.svg new file mode 100644 index 00000000..dc67f4d5 --- /dev/null +++ b/demo/src/assets/pdf.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/transcription.svg b/demo/src/assets/transcription.svg new file mode 100644 index 00000000..8b887a6f --- /dev/null +++ b/demo/src/assets/transcription.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/demo/src/assets/voice.svg b/demo/src/assets/voice.svg new file mode 100644 index 00000000..86a880b0 --- /dev/null +++ b/demo/src/assets/voice.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/common/constant.ts b/demo/src/common/constant.ts new file mode 100644 index 00000000..fee0c18e --- /dev/null +++ b/demo/src/common/constant.ts @@ -0,0 +1,88 @@ +import { IOptions, ColorItem, LanguageOptionItem, VoiceOptionItem, GraphOptionItem } from "@/types" +export const GITHUB_URL = "https://github.com/TEN-framework/ASTRA.ai" +export const OPTIONS_KEY = "__options__" +export const DEFAULT_OPTIONS: IOptions = { + channel: "", + userName: "", + userId: 0 +} +export const DESCRIPTION = "This is an AI voice assistant powered by ASTRA.ai framework, Agora, Azure and ChatGPT." +export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [ + { + label: "English", + value: "en-US" + }, + { + label: "Chinese", + value: "zh-CN" + }, + { + label: "Korean", + value: "ko-KR" + }, + { + label: "Japanese", + value: "ja-JP" + } +] +export const GRAPH_OPTIONS: GraphOptionItem[] = [ + { + label: "Voice Agent - OpenAI LLM + Azure TTS", + value: "va.openai.azure" + }, + { + label: "Voice Agent with Vision - OpenAI LLM + Azure TTS", + value: "camera.va.openai.azure" + }, + { + label: "Voice Agent with Knowledge - RAG + Qwen LLM + Cosy TTS", + value: "va.qwen.rag" + }, +] + +export const isRagGraph = (graphName: string) => { + return graphName === "va.qwen.rag" +} + +export const VOICE_OPTIONS: VoiceOptionItem[] = [ + { + label: "Male", + value: "male" + }, + { + label: "Female", + value: "female" + } +] +export const COLOR_LIST: ColorItem[] = [{ + active: "#0888FF", + default: "#143354" +}, { + active: "#563FD8", + default: "#2C2553" +}, +{ + active: "#18A957", + default: "#173526" +}, { + active: "#FFAB08", + default: "#423115" +}, { + active: "#FD5C63", + default: "#462629" +}, { + active: "#E225B2", + default: "#481C3F" +}] + +export type VoiceTypeMap = { + [voiceType: string]: string; +}; + +export type VendorNameMap = { + [vendorName: string]: VoiceTypeMap; +}; + +export type LanguageMap = { + [language: string]: VendorNameMap; +}; \ No newline at end of file diff --git a/demo/src/common/hooks.ts b/demo/src/common/hooks.ts new file mode 100644 index 00000000..9759fa29 --- /dev/null +++ b/demo/src/common/hooks.ts @@ -0,0 +1,131 @@ +"use client" + +import { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" +import { normalizeFrequencies } from "./utils" +import { useState, useEffect, useMemo, useRef } from "react" +import type { AppDispatch, AppStore, RootState } from "../store" +import { useDispatch, useSelector, useStore } from "react-redux" +import { Grid } from "antd" + +const { useBreakpoint } = Grid; + +export const useAppDispatch = useDispatch.withTypes() +export const useAppSelector = useSelector.withTypes() +export const useAppStore = useStore.withTypes() + +export const useMultibandTrackVolume = ( + track?: IMicrophoneAudioTrack | MediaStreamTrack, + bands: number = 5, + loPass: number = 100, + hiPass: number = 600 +) => { + const [frequencyBands, setFrequencyBands] = useState([]); + + useEffect(() => { + if (!track) { + return setFrequencyBands(new Array(bands).fill(new Float32Array(0))) + } + + const ctx = new AudioContext(); + let finTrack = track instanceof MediaStreamTrack ? track : track.getMediaStreamTrack() + const mediaStream = new MediaStream([finTrack]); + const source = ctx.createMediaStreamSource(mediaStream); + const analyser = ctx.createAnalyser(); + analyser.fftSize = 2048 + + source.connect(analyser); + + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Float32Array(bufferLength); + + const updateVolume = () => { + analyser.getFloatFrequencyData(dataArray); + let frequencies: Float32Array = new Float32Array(dataArray.length); + for (let i = 0; i < dataArray.length; i++) { + frequencies[i] = dataArray[i]; + } + frequencies = frequencies.slice(loPass, hiPass); + + const normalizedFrequencies = normalizeFrequencies(frequencies); + const chunkSize = Math.ceil(normalizedFrequencies.length / bands); + const chunks: Float32Array[] = []; + for (let i = 0; i < bands; i++) { + chunks.push( + normalizedFrequencies.slice(i * chunkSize, (i + 1) * chunkSize) + ); + } + + setFrequencyBands(chunks); + }; + + const interval = setInterval(updateVolume, 10); + + return () => { + source.disconnect(); + clearInterval(interval); + }; + }, [track, loPass, hiPass, bands]); + + return frequencyBands; +}; + +export const useAutoScroll = (ref: React.RefObject) => { + + const callback: MutationCallback = (mutationList, observer) => { + mutationList.forEach((mutation) => { + switch (mutation.type) { + case "childList": + if (!ref.current) { + return + } + ref.current.scrollTop = ref.current.scrollHeight; + break; + } + }) + } + + useEffect(() => { + if (!ref.current) { + return; + } + const observer = new MutationObserver(callback); + observer.observe(ref.current, { + childList: true, + subtree: true + }); + + return () => { + observer.disconnect(); + }; + }, [ref]); +} + +export const useSmallScreen = () => { + const screens = useBreakpoint(); + + const xs = useMemo(() => { + return !screens.sm && screens.xs + }, [screens]) + + const sm = useMemo(() => { + return !screens.md && screens.sm + }, [screens]) + + return { + xs, + sm, + isSmallScreen: xs || sm + } +} + +export const usePrevious = (value: any) => { + const ref = useRef(); + + useEffect(() => { + ref.current = value; + }, [value]); + + return ref.current; +}; + + diff --git a/demo/src/common/index.ts b/demo/src/common/index.ts new file mode 100644 index 00000000..3c2b0300 --- /dev/null +++ b/demo/src/common/index.ts @@ -0,0 +1,6 @@ +export * from "./hooks" +export * from "./constant" +export * from "./utils" +export * from "./storage" +export * from "./request" +export * from "./mock" diff --git a/demo/src/common/mock.ts b/demo/src/common/mock.ts new file mode 100644 index 00000000..db1e2ff8 --- /dev/null +++ b/demo/src/common/mock.ts @@ -0,0 +1,41 @@ +import { getRandomUserId } from "./utils" +import { IChatItem } from "@/types" + + +const SENTENCES = [ + "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium.", + "Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit.", + "Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit.", + "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.", + "Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.", + "Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", +] + + +export const genRandomParagraph = (num: number = 0): string => { + let paragraph = "" + for (let i = 0; i < num; i++) { + const randomIndex = Math.floor(Math.random() * SENTENCES.length) + paragraph += SENTENCES[randomIndex] + " " + } + + return paragraph.trim() +} + + +export const genRandomChatList = (num: number = 10): IChatItem[] => { + const arr: IChatItem[] = [] + for (let i = 0; i < num; i++) { + const type = Math.random() > 0.5 ? "agent" : "user" + arr.push({ + userId: getRandomUserId(), + userName: type == "agent" ? "Agent" : "You", + text: genRandomParagraph(3), + type, + time: Date.now(), + }) + } + + return arr +} diff --git a/demo/src/common/request.ts b/demo/src/common/request.ts new file mode 100644 index 00000000..0c65e352 --- /dev/null +++ b/demo/src/common/request.ts @@ -0,0 +1,99 @@ +import { genUUID } from "./utils" +import { Language } from "@/types" +import axios from "axios" + +interface StartRequestConfig { + channel: string + userId: number, + graphName: string, + language: Language, + voiceType: "male" | "female" +} + +interface GenAgoraDataConfig { + userId: string | number + channel: string +} + +export const apiGenAgoraData = async (config: GenAgoraDataConfig) => { + // the request will be rewrite at next.config.mjs to send to $AGENT_SERVER_URL + const url = `/api/token/generate` + const { userId, channel } = config + const data = { + request_id: genUUID(), + uid: userId, + channel_name: channel + } + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} + return resp +} + +export const apiStartService = async (config: StartRequestConfig): Promise => { + // look at app/api/agents/start/route.tsx for the server-side implementation + const url = `/api/agents/start` + const { channel, userId, graphName, language, voiceType } = config + const data = { + request_id: genUUID(), + channel_name: channel, + user_uid: userId, + graph_name: graphName, + language, + voice_type: voiceType + } + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} + return resp +} + +export const apiStopService = async (channel: string) => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/agents/stop` + const data = { + request_id: genUUID(), + channel_name: channel + } + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} + return resp +} + +export const apiGetDocumentList = async () => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/vector/document/preset/list` + let resp: any = await axios.get(url) + resp = (resp.data) || {} + if (resp.code !== "0") { + throw new Error(resp.msg) + } + return resp +} + +export const apiUpdateDocument = async (options: { channel: string, collection: string, fileName: string }) => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/vector/document/update` + const { channel, collection, fileName } = options + const data = { + request_id: genUUID(), + channel_name: channel, + collection: collection, + file_name: fileName + } + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} + return resp +} + + +// ping/pong +export const apiPing = async (channel: string) => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/agents/ping` + const data = { + request_id: genUUID(), + channel_name: channel + } + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} + return resp +} diff --git a/demo/src/common/storage.ts b/demo/src/common/storage.ts new file mode 100644 index 00000000..ed96083d --- /dev/null +++ b/demo/src/common/storage.ts @@ -0,0 +1,21 @@ +import { IOptions } from "@/types" +import { OPTIONS_KEY, DEFAULT_OPTIONS } from "./constant" + +export const getOptionsFromLocal = () => { + if (typeof window !== "undefined") { + const data = localStorage.getItem(OPTIONS_KEY) + if (data) { + return JSON.parse(data) + } + } + return DEFAULT_OPTIONS +} + + +export const setOptionsToLocal = (options: IOptions) => { + if (typeof window !== "undefined") { + localStorage.setItem(OPTIONS_KEY, JSON.stringify(options)) + } +} + + diff --git a/demo/src/common/utils.ts b/demo/src/common/utils.ts new file mode 100644 index 00000000..1d6f0d00 --- /dev/null +++ b/demo/src/common/utils.ts @@ -0,0 +1,59 @@ +export const genRandomString = (length: number = 10) => { + let result = ''; + const characters = 'abcdefghijklmnopqrstuvwxyz0123456789'; + const charactersLength = characters.length; + + for (let i = 0; i < length; i++) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + } + + return result; +} + + +export const getRandomUserId = (): number => { + return Math.floor(Math.random() * 99999) + 100000 +} + +export const getRandomChannel = (number = 6) => { + return "agora_" + genRandomString(number) +} + + +export const sleep = (ms: number) => { + return new Promise(resolve => setTimeout(resolve, ms)); +} + + +export const normalizeFrequencies = (frequencies: Float32Array) => { + const normalizeDb = (value: number) => { + const minDb = -100; + const maxDb = -10; + let db = 1 - (Math.max(minDb, Math.min(maxDb, value)) * -1) / 100; + db = Math.sqrt(db); + + return db; + }; + + // Normalize all frequency values + return frequencies.map((value) => { + if (value === -Infinity) { + return 0; + } + return normalizeDb(value); + }); +}; + + +export const genUUID = () => { + return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function (c) { + const r = (Math.random() * 16) | 0 + const v = c === "x" ? r : (r & 0x3) | 0x8 + return v.toString(16) + }) +} + + +export const isMobile = () => { + return /Mobile|iPhone|iPad|Android|Windows Phone/i.test(navigator.userAgent) +} \ No newline at end of file diff --git a/demo/src/components/authInitializer/index.tsx b/demo/src/components/authInitializer/index.tsx new file mode 100644 index 00000000..5ef763a1 --- /dev/null +++ b/demo/src/components/authInitializer/index.tsx @@ -0,0 +1,29 @@ +"use client" + +import { ReactNode, useEffect } from "react" +import { useAppDispatch, getOptionsFromLocal } from "@/common" +import { setOptions, reset } from "@/store/reducers/global" + +interface AuthInitializerProps { + children: ReactNode; +} + +const AuthInitializer = (props: AuthInitializerProps) => { + const { children } = props; + const dispatch = useAppDispatch() + + useEffect(() => { + if (typeof window !== "undefined") { + const options = getOptionsFromLocal() + if (options) { + dispatch(reset()) + dispatch(setOptions(options)) + } + } + }, [dispatch]) + + return children +} + + +export default AuthInitializer; diff --git a/demo/src/components/customSelect/index.module.scss b/demo/src/components/customSelect/index.module.scss new file mode 100644 index 00000000..0649e994 --- /dev/null +++ b/demo/src/components/customSelect/index.module.scss @@ -0,0 +1,22 @@ +.selectWrapper { + position: relative; + + .prefixIconWrapper { + position: absolute; + z-index: 1; + width: 3rem; + height: 100%; + display: flex; + align-items: center; + justify-content: center; + } + + :global(.customSelect) { + width: 100%; + + :global(.ant-select-selector) { + padding-left: calc(3rem - 8px) !important; + } + } + +} diff --git a/demo/src/components/customSelect/index.tsx b/demo/src/components/customSelect/index.tsx new file mode 100644 index 00000000..8dd1b188 --- /dev/null +++ b/demo/src/components/customSelect/index.tsx @@ -0,0 +1,19 @@ +import { Select, SelectProps } from "antd" +import styles from "./index.module.scss" + +type CustomSelectProps = SelectProps & { + prefixIcon?: React.ReactNode; +} + +const CustomSelect = (props: CustomSelectProps) => { + + const { prefixIcon, className, ...rest } = props; + + return
+ {prefixIcon &&
{prefixIcon}
} + +
+} + + +export default CustomSelect diff --git a/demo/src/components/icons/cam/index.tsx b/demo/src/components/icons/cam/index.tsx new file mode 100644 index 00000000..628e651c --- /dev/null +++ b/demo/src/components/icons/cam/index.tsx @@ -0,0 +1,17 @@ +import camMuteSvg from "@/assets/cam_mute.svg" +import camUnMuteSvg from "@/assets/cam_unmute.svg" +import { IconProps } from "../types" + +interface ICamIconProps extends IconProps { + active?: boolean +} + +export const CamIcon = (props: ICamIconProps) => { + const { active, ...rest } = props + + if (active) { + return camUnMuteSvg(rest) + } else { + return camMuteSvg(rest) + } +} diff --git a/demo/src/components/icons/colorPicker/index.tsx b/demo/src/components/icons/colorPicker/index.tsx new file mode 100644 index 00000000..81efcb12 --- /dev/null +++ b/demo/src/components/icons/colorPicker/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import ColorPickerSvg from "@/assets/color_picker.svg" + +export const ColorPickerIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/github/index.tsx b/demo/src/components/icons/github/index.tsx new file mode 100644 index 00000000..cee01b8b --- /dev/null +++ b/demo/src/components/icons/github/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import GithubSvg from "@/assets/github.svg" + +export const GithubIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/index.tsx b/demo/src/components/icons/index.tsx new file mode 100644 index 00000000..e303674a --- /dev/null +++ b/demo/src/components/icons/index.tsx @@ -0,0 +1,10 @@ +export * from "./mic" +export * from "./cam" +export * from "./network" +export * from "./github" +export * from "./transcription" +export * from "./logo" +export * from "./info" +export * from "./colorPicker" +export * from "./voice" +export * from "./pdf" diff --git a/demo/src/components/icons/info/index.tsx b/demo/src/components/icons/info/index.tsx new file mode 100644 index 00000000..cf783be9 --- /dev/null +++ b/demo/src/components/icons/info/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import InfoSvg from "@/assets/info.svg" + +export const InfoIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/logo/index.tsx b/demo/src/components/icons/logo/index.tsx new file mode 100644 index 00000000..f86d5246 --- /dev/null +++ b/demo/src/components/icons/logo/index.tsx @@ -0,0 +1,8 @@ +import { IconProps } from "../types" +import LogoSvg from "@/assets/logo.svg" +import SmallLogoSvg from "@/assets/logo_small.svg" + +export const LogoIcon = (props: IconProps) => { + const { size = "default" } = props + return size == "small" ? : +} diff --git a/demo/src/components/icons/mic/index.tsx b/demo/src/components/icons/mic/index.tsx new file mode 100644 index 00000000..0a693033 --- /dev/null +++ b/demo/src/components/icons/mic/index.tsx @@ -0,0 +1,23 @@ +import { IconProps } from "../types" +import micMuteSvg from "@/assets/mic_mute.svg" +import micUnMuteSvg from "@/assets/mic_unmute.svg" + +interface IMicIconProps extends IconProps { + active?: boolean +} + +export const MicIcon = (props: IMicIconProps) => { + const { active, color, ...rest } = props + + if (active) { + return micUnMuteSvg({ + color: color || "#3D53F5", + ...rest, + }) + } else { + return micMuteSvg({ + color: color || "#667085", + ...rest, + }) + } +} diff --git a/demo/src/components/icons/network/index.tsx b/demo/src/components/icons/network/index.tsx new file mode 100644 index 00000000..1950cda7 --- /dev/null +++ b/demo/src/components/icons/network/index.tsx @@ -0,0 +1,33 @@ +import averageSvg from "@/assets/network/average.svg" +import goodSvg from "@/assets/network/good.svg" +import poorSvg from "@/assets/network/poor.svg" +import disconnectedSvg from "@/assets/network/disconnected.svg" +import excellentSvg from "@/assets/network/excellent.svg" + +import { IconProps } from "../types" + +interface INetworkIconProps extends IconProps { + level?: number +} + +export const NetworkIcon = (props: INetworkIconProps) => { + const { level, ...rest } = props + switch (level) { + case 0: + return disconnectedSvg(rest) + case 1: + return excellentSvg(rest) + case 2: + return goodSvg(rest) + case 3: + return averageSvg(rest) + case 4: + return averageSvg(rest) + case 5: + return poorSvg(rest) + case 6: + return disconnectedSvg(rest) + default: + return disconnectedSvg(rest) + } +} diff --git a/demo/src/components/icons/pdf/index.tsx b/demo/src/components/icons/pdf/index.tsx new file mode 100644 index 00000000..83de8b2d --- /dev/null +++ b/demo/src/components/icons/pdf/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import PdfSvg from "@/assets/pdf.svg" + +export const PdfIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/transcription/index.tsx b/demo/src/components/icons/transcription/index.tsx new file mode 100644 index 00000000..757adbce --- /dev/null +++ b/demo/src/components/icons/transcription/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import TranscriptionSvg from "@/assets/transcription.svg" + +export const TranscriptionIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/types.ts b/demo/src/components/icons/types.ts new file mode 100644 index 00000000..c37e8133 --- /dev/null +++ b/demo/src/components/icons/types.ts @@ -0,0 +1,10 @@ +export interface IconProps { + width?: number + height?: number + color?: string + viewBox?: string + size?: "small" | "default" + // style?: React.CSSProperties + transform?: string + onClick?: () => void +} diff --git a/demo/src/components/icons/voice/index.tsx b/demo/src/components/icons/voice/index.tsx new file mode 100644 index 00000000..87164cea --- /dev/null +++ b/demo/src/components/icons/voice/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import VoiceSvg from "@/assets/voice.svg" + +export const VoiceIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/loginCard/index.module.scss b/demo/src/components/loginCard/index.module.scss new file mode 100644 index 00000000..966ebc20 --- /dev/null +++ b/demo/src/components/loginCard/index.module.scss @@ -0,0 +1,112 @@ +.card { + position: absolute; + top: 45%; + left: 50%; + transform: translate(-50%, -50%); + display: flex; + width: 368px; + padding: 100px 24px 40px 24px; + flex-direction: column; + justify-content: center; + align-items: center; + border-radius: 20px; + border: 1px solid #20272D; + background: linear-gradient(154deg, rgba(31, 69, 141, 0.16) 0%, rgba(31, 69, 141, 0.00) 30%), linear-gradient(153deg, rgba(31, 54, 97, 0.00) 53.75%, #1F458D 100%), rgba(15, 15, 17, 0.10); + box-shadow: 0px 3.999px 48.988px 0px rgba(0, 7, 72, 0.12); + backdrop-filter: blur(8.8px); + + .top { + .github { + position: absolute; + right: 24px; + top: 24px; + display: flex; + padding: 4px 8px 4px 4px; + align-items: center; + gap: 4px; + border-radius: 100px; + border: 1px solid #2B2F36; + cursor: pointer; + + .text { + color: var(--Grey-300, #EAECF0); + font-size: 12px; + line-height: 150%; + } + } + } + + .content { + + .title { + margin-bottom: 32px; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 12px; + + .text { + margin-top: 8px; + color: var(--Grey-300, #EAECF0); + text-align: center; + font-size: 18px; + font-weight: 500; + } + } + + .section { + + input { + display: flex; + width: 320px; + flex-direction: column; + align-items: flex-start; + gap: 6px; + display: flex; + height: 38px; + padding: 12px 8px; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 6px; + border: 1px solid #2B2F36; + box-shadow: 0px 4.282px 52.456px 0px rgba(0, 7, 72, 0.12); + backdrop-filter: blur(13px); + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.20); + } + + .btn { + display: flex; + padding: 10px 18px; + justify-content: center; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 8px; + background: var(--primary-500-base, #0888FF); + box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05); + cursor: pointer; + + .btnText { + color: var(---white, #FFF); + font-size: 16px; + font-weight: 500; + line-height: 24px; + } + } + } + + .section+.section { + margin-top: 24px; + } + + .version { + text-align: center; + margin-top: 32px; + color: var(--Grey-600, #667085); + line-height: 22px; + } + + } +} diff --git a/demo/src/components/loginCard/index.tsx b/demo/src/components/loginCard/index.tsx new file mode 100644 index 00000000..56a78986 --- /dev/null +++ b/demo/src/components/loginCard/index.tsx @@ -0,0 +1,77 @@ +"use client" + +import packageData from "../../../package.json" +import { useRouter } from 'next/navigation' +import { message } from "antd" +import { useState } from "react" +import { GithubIcon, LogoIcon } from "../icons" +import { GITHUB_URL, getRandomUserId, useAppDispatch, getRandomChannel } from "@/common" +import { setOptions } from "@/store/reducers/global" +import styles from "./index.module.scss" + + +const { version } = packageData + +const LoginCard = () => { + const dispatch = useAppDispatch() + const router = useRouter() + const [userName, setUserName] = useState("") + + const onClickGithub = () => { + if (typeof window !== "undefined") { + window.open(GITHUB_URL, "_blank") + } + } + + const onUserNameChange = (e: any) => { + let value = e.target.value + value = value.replace(/\s/g, ""); + setUserName(value) + } + + + + const onClickJoin = () => { + if (!userName) { + message.error("please input user name") + return + } + const userId = getRandomUserId() + dispatch(setOptions({ + userName, + channel: getRandomChannel(), + userId + })) + router.push("/home") + } + + + return
+
+ + + GitHub + +
+
+
+ + Astra - a multimodal interactive agent +
+
+ +
+
+
+ Join +
+
+
Version {version}
+
+
+ + + return +} + +export default LoginCard diff --git a/demo/src/components/pdfSelect/index.module.scss b/demo/src/components/pdfSelect/index.module.scss new file mode 100644 index 00000000..adb93280 --- /dev/null +++ b/demo/src/components/pdfSelect/index.module.scss @@ -0,0 +1,8 @@ +// .pdfSelect { + // min-width: 200px; + // max-width: 300px; + // } +.dropdownRender { + display: flex; + justify-content: flex-end; +} diff --git a/demo/src/components/pdfSelect/index.tsx b/demo/src/components/pdfSelect/index.tsx new file mode 100644 index 00000000..f593bf1d --- /dev/null +++ b/demo/src/components/pdfSelect/index.tsx @@ -0,0 +1,88 @@ +import { ReactElement, useState } from "react" +import { PdfIcon } from "@/components/icons" +import CustomSelect from "@/components/customSelect" +import { Divider, message } from 'antd'; +import { useEffect } from 'react'; +import { apiGetDocumentList, apiUpdateDocument, useAppSelector } from "@/common" +import PdfUpload from "./upload" +import { OptionType, IPdfData } from "@/types" + +import styles from "./index.module.scss" + +const PdfSelect = () => { + const options = useAppSelector(state => state.global.options) + const { channel } = options + const [pdfOptions, setPdfOptions] = useState([]) + const [selectedPdf, setSelectedPdf] = useState('') + const agentConnected = useAppSelector(state => state.global.agentConnected) + + + useEffect(() => { + if(agentConnected) { + getPDFOptions() + } else { + setPdfOptions([{ + value: '', + label: 'Please select a PDF file' + }]) + } + }, [agentConnected]) + + + const getPDFOptions = async () => { + const res = await apiGetDocumentList() + setPdfOptions([{ + value: '', + label: 'Please select a PDF file' + }].concat(res.data.map((item: any) => { + return { + value: item.collection, + label: item.file_name + } + }))) + setSelectedPdf('') + } + + const onUploadSuccess = (data: IPdfData) => { + setPdfOptions([...pdfOptions, { + value: data.collection, + label: data.fileName + }]) + setSelectedPdf(data.collection) + } + + const pdfDropdownRender = (menu: ReactElement) => { + return <> + {menu} + +
+ +
+ + } + + + const onSelectPdf = async (val: string) => { + const item = pdfOptions.find(item => item.value === val) + if (!item) { + return message.error("Please select a PDF file") + } + setSelectedPdf(val) + await apiUpdateDocument({ + collection: val, + fileName: item.label, + channel + }) + } + + + return } + onChange={onSelectPdf} + value={selectedPdf} + options={pdfOptions} + dropdownRender={pdfDropdownRender} + className={styles.pdfSelect} placeholder="Select a PDF file"> +} + +export default PdfSelect diff --git a/demo/src/components/pdfSelect/upload/index.module.scss b/demo/src/components/pdfSelect/upload/index.module.scss new file mode 100644 index 00000000..fd559b5e --- /dev/null +++ b/demo/src/components/pdfSelect/upload/index.module.scss @@ -0,0 +1,7 @@ +.btn { + color: var(--theme-color, #EAECF0); + + &:hover { + color: var(--theme-color, #EAECF0) !important; + } +} diff --git a/demo/src/components/pdfSelect/upload/index.tsx b/demo/src/components/pdfSelect/upload/index.tsx new file mode 100644 index 00000000..4eab9684 --- /dev/null +++ b/demo/src/components/pdfSelect/upload/index.tsx @@ -0,0 +1,75 @@ +import { Select, Button, message, Upload, UploadProps } from "antd" +import { useState } from "react" +import { PlusOutlined, LoadingOutlined } from '@ant-design/icons'; +import { useAppSelector, genUUID } from "@/common" +import { IPdfData } from "@/types" + +import styles from "./index.module.scss" + +interface PdfSelectProps { + onSuccess?: (data: IPdfData) => void +} + +const PdfUpload = (props: PdfSelectProps) => { + const { onSuccess } = props + const agentConnected = useAppSelector(state => state.global.agentConnected) + const options = useAppSelector(state => state.global.options) + const { channel, userId } = options + + const [uploading, setUploading] = useState(false) + + const uploadProps: UploadProps = { + accept: "application/pdf", + maxCount: 1, + showUploadList: false, + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + action: `/api/vector/document/upload`, + data: { + channel_name: channel, + uid: String(userId), + request_id: genUUID() + }, + onChange: (info) => { + const { file } = info + const { status, name } = file + if (status == "uploading") { + setUploading(true) + } else if (status == 'done') { + setUploading(false) + const { response } = file + if (response.code == "0") { + message.success(`Upload ${name} success`) + const { collection, file_name } = response.data + onSuccess && onSuccess({ + fileName: file_name, + collection + }) + } else { + message.error(response.msg) + } + } else if (status == 'error') { + setUploading(false) + message.error(`Upload ${name} failed`) + } + } + } + + const onClickUploadPDF = (e: any) => { + if (!agentConnected) { + message.error("Please connect to agent first") + e.stopPropagation() + } + } + + + return + + +} + + +export default PdfUpload diff --git a/demo/src/manager/events.ts b/demo/src/manager/events.ts new file mode 100644 index 00000000..055c6d87 --- /dev/null +++ b/demo/src/manager/events.ts @@ -0,0 +1,51 @@ +import { EventHandler } from "./types" + +export class AGEventEmitter { + private readonly _eventMap: Map[]> = new Map() + + once(evt: Key, cb: T[Key]) { + const wrapper = (...args: any[]) => { + this.off(evt, wrapper as any) + ;(cb as any)(...args) + } + this.on(evt, wrapper as any) + return this + } + + on(evt: Key, cb: T[Key]) { + const cbs = this._eventMap.get(evt) ?? [] + cbs.push(cb as any) + this._eventMap.set(evt, cbs) + return this + } + + off(evt: Key, cb: T[Key]) { + const cbs = this._eventMap.get(evt) + if (cbs) { + this._eventMap.set( + evt, + cbs.filter((it) => it !== cb), + ) + } + return this + } + + removeAllEventListeners(): void { + this._eventMap.clear() + } + + emit(evt: Key, ...args: any[]) { + const cbs = this._eventMap.get(evt) ?? [] + for (const cb of cbs) { + try { + cb && cb(...args) + } catch (e) { + // cb exception should not affect other callbacks + const error = e as Error + const details = error.stack || error.message + console.error(`[event] handling event ${evt.toString()} fail: ${details}`) + } + } + return this + } +} diff --git a/demo/src/manager/index.ts b/demo/src/manager/index.ts new file mode 100644 index 00000000..fa9dfbf2 --- /dev/null +++ b/demo/src/manager/index.ts @@ -0,0 +1 @@ +export * from "./rtc" diff --git a/demo/src/manager/rtc/index.ts b/demo/src/manager/rtc/index.ts new file mode 100644 index 00000000..e9fd6272 --- /dev/null +++ b/demo/src/manager/rtc/index.ts @@ -0,0 +1,2 @@ +export * from "./rtc" +export * from "./types" diff --git a/demo/src/manager/rtc/rtc.ts b/demo/src/manager/rtc/rtc.ts new file mode 100644 index 00000000..4139d89e --- /dev/null +++ b/demo/src/manager/rtc/rtc.ts @@ -0,0 +1,199 @@ +"use client" + +import protoRoot from "@/protobuf/SttMessage_es6.js" +import AgoraRTC, { + IAgoraRTCClient, + IMicrophoneAudioTrack, + IRemoteAudioTrack, + UID, +} from "agora-rtc-sdk-ng" +import { ITextItem } from "@/types" +import { AGEventEmitter } from "../events" +import { RtcEvents, IUserTracks } from "./types" +import { apiGenAgoraData } from "@/common" + +export class RtcManager extends AGEventEmitter { + private _joined + client: IAgoraRTCClient + localTracks: IUserTracks + + constructor() { + super() + this._joined = false + this.localTracks = {} + this.client = AgoraRTC.createClient({ mode: "rtc", codec: "vp8" }) + this._listenRtcEvents() + } + + async join({ channel, userId }: { channel: string; userId: number }) { + if (!this._joined) { + const res = await apiGenAgoraData({ channel, userId }) + const { code, data } = res + if (code != 0) { + throw new Error("Failed to get Agora token") + } + const { appId, token } = data + await this.client?.join(appId, channel, token, userId) + this._joined = true + } + } + + async createTracks() { + try { + const videoTrack = await AgoraRTC.createCameraVideoTrack() + this.localTracks.videoTrack = videoTrack + } catch (err) { + console.error("Failed to create video track", err) + } + try { + const audioTrack = await AgoraRTC.createMicrophoneAudioTrack() + this.localTracks.audioTrack = audioTrack + } catch (err) { + console.error("Failed to create audio track", err) + } + this.emit("localTracksChanged", this.localTracks) + } + + async publish() { + const tracks = [] + if (this.localTracks.videoTrack) { + tracks.push(this.localTracks.videoTrack) + } + if (this.localTracks.audioTrack) { + tracks.push(this.localTracks.audioTrack) + } + if (tracks.length) { + await this.client.publish(tracks) + } + } + + async destroy() { + this.localTracks?.audioTrack?.close() + this.localTracks?.videoTrack?.close() + if (this._joined) { + await this.client?.leave() + } + this._resetData() + } + + // ----------- public methods ------------ + + // -------------- private methods -------------- + private _listenRtcEvents() { + this.client.on("network-quality", (quality) => { + this.emit("networkQuality", quality) + }) + this.client.on("user-published", async (user, mediaType) => { + await this.client.subscribe(user, mediaType) + if (mediaType === "audio") { + this._playAudio(user.audioTrack) + } + this.emit("remoteUserChanged", { + userId: user.uid, + audioTrack: user.audioTrack, + videoTrack: user.videoTrack, + }) + }) + this.client.on("user-unpublished", async (user, mediaType) => { + await this.client.unsubscribe(user, mediaType) + this.emit("remoteUserChanged", { + userId: user.uid, + audioTrack: user.audioTrack, + videoTrack: user.videoTrack, + }) + }) + this.client.on("stream-message", (uid: UID, stream: any) => { + this._parseData(stream) + }) + } + + private _parseData(data: any): ITextItem | void { + let decoder = new TextDecoder('utf-8'); + let decodedMessage = decoder.decode(data); + const textstream = JSON.parse(decodedMessage); + + console.log("[test] textstream raw data", JSON.stringify(textstream)); + + const { stream_id, is_final, text, text_ts, data_type, message_id, part_number, total_parts } = textstream; + + if (total_parts > 0) { + // If message is split, handle it accordingly + this._handleSplitMessage(message_id, part_number, total_parts, stream_id, is_final, text, text_ts); + } else { + // If there is no message_id, treat it as a complete message + this._handleCompleteMessage(stream_id, is_final, text, text_ts); + } + } + + private messageCache: { [key: string]: { parts: string[], totalParts: number } } = {}; + + /** + * Handle complete messages (not split). + */ + private _handleCompleteMessage(stream_id: number, is_final: boolean, text: string, text_ts: number): void { + const textItem: ITextItem = { + uid: `${stream_id}`, + time: text_ts, + dataType: "transcribe", + text: text, + isFinal: is_final + }; + + if (text.trim().length > 0) { + this.emit("textChanged", textItem); + } + } + + /** + * Handle split messages, track parts, and reassemble once all parts are received. + */ + private _handleSplitMessage( + message_id: string, + part_number: number, + total_parts: number, + stream_id: number, + is_final: boolean, + text: string, + text_ts: number + ): void { + // Ensure the messageCache entry exists for this message_id + if (!this.messageCache[message_id]) { + this.messageCache[message_id] = { parts: [], totalParts: total_parts }; + } + + const cache = this.messageCache[message_id]; + + // Store the received part at the correct index (part_number starts from 1, so we use part_number - 1) + cache.parts[part_number - 1] = text; + + // Check if all parts have been received + const receivedPartsCount = cache.parts.filter(part => part !== undefined).length; + + if (receivedPartsCount === total_parts) { + // All parts have been received, reassemble the message + const fullText = cache.parts.join(''); + + // Now that the message is reassembled, handle it like a complete message + this._handleCompleteMessage(stream_id, is_final, fullText, text_ts); + + // Remove the cached message since it is now fully processed + delete this.messageCache[message_id]; + } + } + + + _playAudio(audioTrack: IMicrophoneAudioTrack | IRemoteAudioTrack | undefined) { + if (audioTrack && !audioTrack.isPlaying) { + audioTrack.play() + } + } + + + private _resetData() { + this.localTracks = {} + this._joined = false + } +} + + +export const rtcManager = new RtcManager() diff --git a/demo/src/manager/rtc/types.ts b/demo/src/manager/rtc/types.ts new file mode 100644 index 00000000..15e3f515 --- /dev/null +++ b/demo/src/manager/rtc/types.ts @@ -0,0 +1,25 @@ +import { + UID, + IAgoraRTCRemoteUser, + IAgoraRTCClient, + ICameraVideoTrack, + IMicrophoneAudioTrack, + NetworkQuality, +} from "agora-rtc-sdk-ng" +import { ITextItem } from "@/types" + +export interface IRtcUser extends IUserTracks { + userId: UID +} + +export interface RtcEvents { + remoteUserChanged: (user: IRtcUser) => void + localTracksChanged: (tracks: IUserTracks) => void + networkQuality: (quality: NetworkQuality) => void + textChanged: (text: ITextItem) => void +} + +export interface IUserTracks { + videoTrack?: ICameraVideoTrack + audioTrack?: IMicrophoneAudioTrack +} diff --git a/demo/src/manager/types.ts b/demo/src/manager/types.ts new file mode 100644 index 00000000..50e5b1c0 --- /dev/null +++ b/demo/src/manager/types.ts @@ -0,0 +1 @@ +export type EventHandler = (...data: T) => void diff --git a/demo/src/middleware.tsx b/demo/src/middleware.tsx new file mode 100644 index 00000000..724e0b4d --- /dev/null +++ b/demo/src/middleware.tsx @@ -0,0 +1,44 @@ +// middleware.js +import { NextRequest, NextResponse } from 'next/server'; + + +const { AGENT_SERVER_URL } = process.env; + +// Check if environment variables are available +if (!AGENT_SERVER_URL) { + throw "Environment variables AGENT_SERVER_URL are not available"; +} + +export function middleware(req: NextRequest) { + const { pathname } = req.nextUrl; + + if (pathname.startsWith('/api/agents/')) { + if (!pathname.startsWith('/api/agents/start')) { + + // Proxy all other agents API requests + const url = req.nextUrl.clone(); + url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/agents/', '/')}`; + + // console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + } + } else if (pathname.startsWith('/api/vector/')) { + + // Proxy all other documents requests + const url = req.nextUrl.clone(); + url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/vector/', '/vector/')}`; + + // console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + } else if (pathname.startsWith('/api/token/')) { + // Proxy all other documents requests + const url = req.nextUrl.clone(); + url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/token/', '/token/')}`; + + // console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + } else { + return NextResponse.next(); + } + +} \ No newline at end of file diff --git a/demo/src/platform/mobile/chat/chatItem/index.module.scss b/demo/src/platform/mobile/chat/chatItem/index.module.scss new file mode 100644 index 00000000..27057120 --- /dev/null +++ b/demo/src/platform/mobile/chat/chatItem/index.module.scss @@ -0,0 +1,86 @@ +.agentChatItem { + width: 100%; + display: flex; + justify-content: flex-start; + + .left { + flex: 0 0 auto; + display: flex; + width: 32px; + height: 32px; + padding: 10px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + border-radius: 200px; + background: var(--Grey-700, #475467); + + .userName { + color: var(---white, #FFF); + text-align: center; + font-size: 14px; + font-weight: 500; + line-height: 150%; + } + } + + .right { + margin-left: 12px; + + .userName { + font-size: 14px; + font-weight: 500; + line-height: 20px; + color: var(--theme-color, #667085) !important; + } + + + .agent { + color: var(--theme-color, #EAECF0) !important; + } + + } +} + +.userChatItem { + width: 100%; + display: flex; + flex-direction: column; + justify-content: flex-end; + align-items: flex-end; + + .userName { + text-align: right; + color: var(--Grey-600, #667085); + font-weight: 500; + line-height: 20px; + } + + + +} + + +.chatItem { + .text { + margin-top: 6px; + color: #FFF; + display: flex; + padding: 8px 14px; + flex-direction: column; + justify-content: left; + font-size: 14px; + font-weight: 400; + line-height: 21px; + white-space: pre-wrap; + border-radius: 0px 8px 8px 8px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} + +.chatItem+.chatItem { + margin-top: 14px; +} diff --git a/demo/src/platform/mobile/chat/chatItem/index.tsx b/demo/src/platform/mobile/chat/chatItem/index.tsx new file mode 100644 index 00000000..bde3350c --- /dev/null +++ b/demo/src/platform/mobile/chat/chatItem/index.tsx @@ -0,0 +1,50 @@ +import { IChatItem } from "@/types" +import styles from "./index.module.scss" + +interface ChatItemProps { + data: IChatItem +} + + +const AgentChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + + return
+ + Ag + + +
Agent
+
+ {text} +
+ +
+} + +const UserChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + return
+
You
+
{text}
+
+} + + +const ChatItem = (props: ChatItemProps) => { + const { data } = props + + + return ( + data.type === "agent" ? : + ); + + +} + + +export default ChatItem diff --git a/demo/src/platform/mobile/chat/index.module.scss b/demo/src/platform/mobile/chat/index.module.scss new file mode 100644 index 00000000..8ded1f2c --- /dev/null +++ b/demo/src/platform/mobile/chat/index.module.scss @@ -0,0 +1,78 @@ +.chat { + flex: 1 1 auto; + display: flex; + flex-direction: column; + align-items: flex-start; + align-self: stretch; + background: #181A1D; + overflow: hidden; + + .header { + display: flex; + flex-direction: column; + align-items: stretch; + row-gap: 10px; + border-bottom: 1px solid #272A2F; + width: 100%; + + + .text { + margin-left: 4px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + height: 40px; + line-height: 40px; + letter-spacing: 0.449px; + } + + .languageSelect { + width: 100%; + } + + + + + } + + .content { + margin-top: 16px; + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 10px; + align-self: stretch; + overflow-y: auto; + + + &::-webkit-scrollbar { + width: 6px + } + + &::-webkit-scrollbar-track { + background-color: transparent; + } + + &::-webkit-scrollbar-thumb { + background-color: #6B6B6B; + border-radius: 4px; + } + } + + +} + + +.dropdownRender { + display: flex; + justify-content: flex-end; + + + .btn { + color: var(--theme-color, #EAECF0); + + &:hover { + color: var(--theme-color, #EAECF0) !important; + } + } +} \ No newline at end of file diff --git a/demo/src/platform/mobile/chat/index.tsx b/demo/src/platform/mobile/chat/index.tsx new file mode 100644 index 00000000..bb071d4e --- /dev/null +++ b/demo/src/platform/mobile/chat/index.tsx @@ -0,0 +1,65 @@ +import { ReactElement, useEffect, useContext, useState } from "react" +import ChatItem from "./chatItem" +import { IChatItem } from "@/types" +import { useAppDispatch, useAutoScroll, LANGUAGE_OPTIONS, useAppSelector, GRAPH_OPTIONS, isRagGraph } from "@/common" +import { setGraphName, setLanguage } from "@/store/reducers/global" +import { Select, } from 'antd'; +import { MenuContext } from "../menu/context" +import PdfSelect from "@/components/pdfSelect" + +import styles from "./index.module.scss" + + +const Chat = () => { + const chatItems = useAppSelector(state => state.global.chatItems) + const language = useAppSelector(state => state.global.language) + const agentConnected = useAppSelector(state => state.global.agentConnected) + const graphName = useAppSelector(state => state.global.graphName) + const dispatch = useAppDispatch() + // genRandomChatList + // const [chatItems, setChatItems] = useState([]) + const context = useContext(MenuContext); + + if (!context) { + throw new Error("MenuContext is not found") + } + + const { scrollToBottom } = context; + + + useEffect(() => { + scrollToBottom() + }, [chatItems, scrollToBottom]) + + + + const onLanguageChange = (val: any) => { + dispatch(setLanguage(val)) + } + + const onGraphNameChange = (val: any) => { + dispatch(setGraphName(val)) + } + + + return
+
+ + + {isRagGraph(graphName) ? : null} +
+
+ {chatItems.map((item, index) => { + return + })} +
+
+} + + +export default Chat diff --git a/demo/src/platform/mobile/description/index.module.scss b/demo/src/platform/mobile/description/index.module.scss new file mode 100644 index 00000000..7305f5a7 --- /dev/null +++ b/demo/src/platform/mobile/description/index.module.scss @@ -0,0 +1,71 @@ +.description { + position: relative; + display: flex; + padding: 12px 16px; + height: 60px; + align-items: center; + gap: 12px; + align-self: stretch; + border-bottom: 1px solid #272A2F; + background: #181A1D; + box-sizing: border-box; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-style: normal; + font-weight: 600; + flex: 1 1 auto; + /* 21px */ + letter-spacing: 0.449px; + } + + .text { + margin-left: 12px; + flex: 1 1 auto; + color: var(--Grey-600, #667085); + font-size: 14px; + font-style: normal; + font-weight: 400; + } + + + .btnConnect { + width: 150px; + display: flex; + padding: 8px 14px; + justify-content: center; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 6px; + background: var(--theme-color, #0888FF); + box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05); + cursor: pointer; + user-select: none; + caret-color: transparent; + box-sizing: border-box; + + .btnText { + color: var(---White, #FFF); + font-size: 14px; + font-weight: 500; + line-height: 20px; + } + + .btnText.disconnect { + color: var(--Error-400-T, #E95C7B); + } + + .loading { + margin-left: 4px; + } + } + + + .btnConnect.disconnect { + background: #181A1D; + border: 1px solid var(--Error-400-T, #E95C7B); + } + +} diff --git a/demo/src/platform/mobile/description/index.tsx b/demo/src/platform/mobile/description/index.tsx new file mode 100644 index 00000000..7473d550 --- /dev/null +++ b/demo/src/platform/mobile/description/index.tsx @@ -0,0 +1,100 @@ +import { setAgentConnected } from "@/store/reducers/global" +import { + DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, + apiStartService, apiStopService +} from "@/common" +import { message } from "antd" +import { useEffect, useState } from "react" +import { LoadingOutlined, } from "@ant-design/icons" +import styles from "./index.module.scss" + +let intervalId: any + +const Description = () => { + const dispatch = useAppDispatch() + const agentConnected = useAppSelector(state => state.global.agentConnected) + const channel = useAppSelector(state => state.global.options.channel) + const userId = useAppSelector(state => state.global.options.userId) + const language = useAppSelector(state => state.global.language) + const voiceType = useAppSelector(state => state.global.voiceType) + const graphName = useAppSelector(state => state.global.graphName) + const [loading, setLoading] = useState(false) + + useEffect(() => { + if (channel) { + checkAgentConnected() + } + }, [channel]) + + + const checkAgentConnected = async () => { + const res: any = await apiPing(channel) + if (res?.code == 0) { + dispatch(setAgentConnected(true)) + } + } + + const onClickConnect = async () => { + if (loading) { + return + } + setLoading(true) + if (agentConnected) { + await apiStopService(channel) + dispatch(setAgentConnected(false)) + message.success("Agent disconnected") + stopPing() + } else { + const res = await apiStartService({ + channel, + userId, + graphName, + language, + voiceType + }) + const { code, msg } = res || {} + if (code != 0) { + if (code == "10001") { + message.error("The number of users experiencing the program simultaneously has exceeded the limit. Please try again later.") + } else { + message.error(`code:${code},msg:${msg}`) + } + setLoading(false) + throw new Error(msg) + } + dispatch(setAgentConnected(true)) + message.success("Agent connected") + startPing() + } + setLoading(false) + } + + const startPing = () => { + if (intervalId) { + stopPing() + } + intervalId = setInterval(() => { + apiPing(channel) + }, 3000) + } + + const stopPing = () => { + if (intervalId) { + clearInterval(intervalId) + intervalId = null + } + } + + return
+ Description + + + {!agentConnected ? "Connect" : "Disconnect"} + {loading ? : null} + + +
+} + + +export default Description diff --git a/demo/src/platform/mobile/entry/index.module.scss b/demo/src/platform/mobile/entry/index.module.scss new file mode 100644 index 00000000..41322c12 --- /dev/null +++ b/demo/src/platform/mobile/entry/index.module.scss @@ -0,0 +1,18 @@ +.entry { + position: relative; + height: 100%; + box-sizing: border-box; + + .content { + position: relative; + padding: 16px; + box-sizing: border-box; + + + .body { + margin-top: 16px; + display: flex; + gap: 24px; + } + } +} diff --git a/demo/src/platform/mobile/entry/index.tsx b/demo/src/platform/mobile/entry/index.tsx new file mode 100644 index 00000000..c5f51d5c --- /dev/null +++ b/demo/src/platform/mobile/entry/index.tsx @@ -0,0 +1,30 @@ +import Chat from "../chat" +import Description from "../description" +import Rtc from "../rtc" +import Header from "../header" +import Menu, { IMenuData } from "../menu" +import styles from "./index.module.scss" + + +const MenuData: IMenuData[] = [{ + name: "Agent", + component: , +}, { + name: "Chat", + component: , +}] + + +const MobileEntry = () => { + + return
+
+ +
+ +
+
+} + + +export default MobileEntry diff --git a/demo/src/platform/mobile/header/index.module.scss b/demo/src/platform/mobile/header/index.module.scss new file mode 100644 index 00000000..707e4215 --- /dev/null +++ b/demo/src/platform/mobile/header/index.module.scss @@ -0,0 +1,57 @@ +.header { + display: flex; + width: 100%; + height: 48px; + padding: 16px; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #24262A; + background: #1E2024; + box-shadow: 0px 12px 16px -4px rgba(8, 15, 52, 0.06), 0px 4px 6px -2px rgba(8, 15, 52, 0.03); + box-sizing: border-box; + z-index: 999; + + .logoWrapper { + display: flex; + align-items: center; + + .text { + margin-left: 8px; + color: var(---white, #FFF); + text-align: right; + font-family: Inter; + font-size: 16px; + font-weight: 500; + } + } + + .content { + padding-left: 12px; + display: flex; + align-items: center; + justify-content: flex-start; + height: 48px; + flex: 1 1 auto; + color: var(--Grey-300, #EAECF0); + font-size: 16px; + font-weight: 500; + line-height: 48px; + letter-spacing: 0.449px; + text-align: center; + + .text { + margin-left: 4px; + font-size: 12px; + } + } + + .links { + display: flex; + align-items: center; + gap: 8px; + + span { + display: flex; + } + } +} diff --git a/demo/src/platform/mobile/header/index.tsx b/demo/src/platform/mobile/header/index.tsx new file mode 100644 index 00000000..6e56a017 --- /dev/null +++ b/demo/src/platform/mobile/header/index.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useAppSelector, GITHUB_URL, useSmallScreen } from "@/common" +import Network from "./network" +import InfoPopover from "./infoPopover" +import StylePopover from "./stylePopover" +import { GithubIcon, LogoIcon, InfoIcon, ColorPickerIcon } from "@/components/icons" + +import styles from "./index.module.scss" + +const Header = () => { + const themeColor = useAppSelector(state => state.global.themeColor) + const options = useAppSelector(state => state.global.options) + const { channel } = options + + + const onClickGithub = () => { + if (typeof window !== "undefined") { + window.open(GITHUB_URL, "_blank") + } + } + + + + return
+ + + + + + + {channel} + + +
+ + + + + + + +
+
+} + + +export default Header diff --git a/demo/src/platform/mobile/header/infoPopover/index.module.scss b/demo/src/platform/mobile/header/infoPopover/index.module.scss new file mode 100644 index 00000000..cd3f72f8 --- /dev/null +++ b/demo/src/platform/mobile/header/infoPopover/index.module.scss @@ -0,0 +1,43 @@ +.info { + display: flex; + padding: 12px 16px; + flex-direction: column; + align-items: flex-start; + gap: 8px; + align-self: stretch; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .item { + width: 100%; + display: flex; + justify-content: space-between; + align-items: center; + + .title { + color: var(--Grey-600, #667085); + font-size: 14px; + font-weight: 400; + line-height: 150%; + } + + .content { + color: var(--theme-color, #FFF); + font-size: 14px; + font-weight: 400; + line-height: 150%; + } + } + + .slider { + height: 1px; + width: 100%; + background-color: #0D0F12; + } +} diff --git a/demo/src/platform/mobile/header/infoPopover/index.tsx b/demo/src/platform/mobile/header/infoPopover/index.tsx new file mode 100644 index 00000000..cd451418 --- /dev/null +++ b/demo/src/platform/mobile/header/infoPopover/index.tsx @@ -0,0 +1,57 @@ +import { useMemo } from "react" +import { useAppSelector } from "@/common" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface InfoPopoverProps { + children?: React.ReactNode +} + +const InfoPopover = (props: InfoPopoverProps) => { + const { children } = props + const options = useAppSelector(state => state.global.options) + const { channel, userId } = options + + const roomConnected = useAppSelector(state => state.global.roomConnected) + const agentConnected = useAppSelector(state => state.global.agentConnected) + + const roomConnectedText = useMemo(() => { + return roomConnected ? "TRUE" : "FALSE" + }, [roomConnected]) + + const agentConnectedText = useMemo(() => { + return agentConnected ? "TRUE" : "FALSE" + }, [agentConnected]) + + + + const content =
+
INFO
+
+ Room + {channel} +
+
+ Participant + {userId} +
+
+
STATUS
+
+
Room connected
+
{roomConnectedText}
+
+
+
Agent connected
+
{agentConnectedText}
+
+
+ + + return {children} + +} + +export default InfoPopover diff --git a/demo/src/platform/mobile/header/network/index.module.scss b/demo/src/platform/mobile/header/network/index.module.scss new file mode 100644 index 00000000..e69de29b diff --git a/demo/src/platform/mobile/header/network/index.tsx b/demo/src/platform/mobile/header/network/index.tsx new file mode 100644 index 00000000..92b4e33b --- /dev/null +++ b/demo/src/platform/mobile/header/network/index.tsx @@ -0,0 +1,37 @@ +"use client"; + +import React from "react"; +import { rtcManager } from "@/manager" +import { NetworkQuality } from "agora-rtc-sdk-ng" +import { useEffect, useState } from "react" +import { NetworkIcon } from "@/components/icons" + +interface NetworkProps { + style?: React.CSSProperties +} + +const NetWork = (props: NetworkProps) => { + const { style } = props + + const [networkQuality, setNetworkQuality] = useState() + + useEffect(() => { + rtcManager.on("networkQuality", onNetworkQuality) + + return () => { + rtcManager.off("networkQuality", onNetworkQuality) + } + }, []) + + const onNetworkQuality = (quality: NetworkQuality) => { + setNetworkQuality(quality) + } + + return ( + + + + ) +} + +export default NetWork diff --git a/demo/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss b/demo/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss new file mode 100644 index 00000000..405e7781 --- /dev/null +++ b/demo/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss @@ -0,0 +1,24 @@ +.colorPicker { + height: 24px; + display: flex; + align-items: center; + + :global(.react-colorful) { + width: 220px; + height: 8px; + } + + :global(.react-colorful__saturation) { + display: none; + } + + :global(.react-colorful__hue) { + border-radius: 8px !important; + height: 8px; + } + + :global(.react-colorful__pointer) { + width: 24px; + height: 24px; + } +} diff --git a/demo/src/platform/mobile/header/stylePopover/colorPicker/index.tsx b/demo/src/platform/mobile/header/stylePopover/colorPicker/index.tsx new file mode 100644 index 00000000..28163d77 --- /dev/null +++ b/demo/src/platform/mobile/header/stylePopover/colorPicker/index.tsx @@ -0,0 +1,22 @@ +"use client" + +import { HexColorPicker } from "react-colorful"; +import { useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import styles from "./index.module.scss"; + +const ColorPicker = () => { + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + const onColorChange = (color: string) => { + console.log(color); + dispatch(setThemeColor(color)) + }; + + return
+ +
+}; + +export default ColorPicker; diff --git a/demo/src/platform/mobile/header/stylePopover/index.module.scss b/demo/src/platform/mobile/header/stylePopover/index.module.scss new file mode 100644 index 00000000..defdcc12 --- /dev/null +++ b/demo/src/platform/mobile/header/stylePopover/index.module.scss @@ -0,0 +1,51 @@ +.info { + padding: 12px 16px; + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 16px; + align-self: stretch; + + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .color { + font-size: 0; + white-space: nowrap; + + .item { + position: relative; + display: inline-block; + width: 28px; + height: 28px; + border-radius: 4px; + border: 2px solid transparent; + font-size: 0; + cursor: pointer; + + .inner { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + width: 18px; + height: 18px; + border-radius: 2px; + box-sizing: border-box; + } + } + + .item+.item { + margin-left: 12px; + } + + } + + +} diff --git a/demo/src/platform/mobile/header/stylePopover/index.tsx b/demo/src/platform/mobile/header/stylePopover/index.tsx new file mode 100644 index 00000000..f8508323 --- /dev/null +++ b/demo/src/platform/mobile/header/stylePopover/index.tsx @@ -0,0 +1,54 @@ +import { useMemo } from "react" +import { COLOR_LIST, useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import ColorPicker from "./colorPicker" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface StylePopoverProps { + children?: React.ReactNode +} + +const StylePopover = (props: StylePopoverProps) => { + const { children } = props + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + + const onClickColor = (index: number) => { + const target = COLOR_LIST[index] + if (target.active !== themeColor) { + dispatch(setThemeColor(target.active)) + } + } + + const content =
+
STYLE
+
+ { + COLOR_LIST.map((item, index) => { + return onClickColor(index)} + className={styles.item} + key={index}> + + + }) + } +
+ +
+ + + return {children} + +} + +export default StylePopover diff --git a/demo/src/platform/mobile/menu/context.ts b/demo/src/platform/mobile/menu/context.ts new file mode 100644 index 00000000..41c52911 --- /dev/null +++ b/demo/src/platform/mobile/menu/context.ts @@ -0,0 +1,9 @@ +import { createContext } from "react" + +export interface MenuContextType { + scrollToBottom: () => void; +} + +export const MenuContext = createContext({ + scrollToBottom: () => { } +}); diff --git a/demo/src/platform/mobile/menu/index.module.scss b/demo/src/platform/mobile/menu/index.module.scss new file mode 100644 index 00000000..58b1b3fe --- /dev/null +++ b/demo/src/platform/mobile/menu/index.module.scss @@ -0,0 +1,69 @@ +.menu { + width: 100%; + border: 1px solid #272A2F; + border-radius: 4px; + background: #0F0F11; + overflow: hidden; + box-sizing: border-box; + + .header { + height: 40px; + overflow: hidden; + border-bottom: 1px solid #272A2F; + box-sizing: border-box; + + .menuItem { + height: 40px; + padding: 0 16px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 40px; + letter-spacing: 0.449px; + display: inline-block; + color: #667085; + background: #181A1E; + cursor: pointer; + border-right: 1px solid #272A2F; + box-sizing: border-box; + overflow: hidden; + background: #0F0F11; + } + + .active { + color: #EAECF0; + background: #181A1D; + } + } + + + .content { + position: relative; + background: #181A1D; + // header 48px + // description 60px + // paddingTop 16px 16px + // menu header 40px + height: calc(100vh - 48px - 60px - 32px - 40px - 2px); + overflow: hidden; + box-sizing: border-box; + + .item { + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; + padding: 16px; + z-index: -1; + overflow: auto; + visibility: hidden; + box-sizing: border-box; + } + + .active { + z-index: 1; + visibility: visible; + } + } +} diff --git a/demo/src/platform/mobile/menu/index.tsx b/demo/src/platform/mobile/menu/index.tsx new file mode 100644 index 00000000..2e20de78 --- /dev/null +++ b/demo/src/platform/mobile/menu/index.tsx @@ -0,0 +1,76 @@ +"use client" + +import { ReactElement, useEffect, useState, useRef, useMemo, useCallback } from "react" +import { useAutoScroll } from "@/common" +import { MenuContext } from "./context" +import styles from "./index.module.scss" + +export interface IMenuData { + name: string, + component: ReactElement +} + +export interface IMenuContentComponentPros { + scrollToBottom: () => void +} + +interface MenuProps { + data: IMenuData[] +} + + +const Menu = (props: MenuProps) => { + const { data } = props + const [activeIndex, setActiveIndex] = useState(0) + const contentRefList = useRef<(HTMLDivElement | null)[]>([]) + + const onClickItem = (index: number) => { + setActiveIndex(index) + } + + useEffect(() => { + scrollToTop() + }, [activeIndex]) + + const scrollToBottom = useCallback(() => { + const current = contentRefList.current?.[activeIndex] + if (current) { + current.scrollTop = current.scrollHeight + } + }, [contentRefList, activeIndex]) + + const scrollToTop = useCallback(() => { + const current = contentRefList.current?.[activeIndex] + if (current) { + current.scrollTop = 0 + } + }, [contentRefList, activeIndex]) + + + return
+
+ {data.map((item, index) => { + return onClickItem(index)}>{item.name} + })} +
+
+ + {data.map((item, index) => { + return
{ + contentRefList.current[index] = el; + }} + className={`${styles.item} ${index == activeIndex ? styles.active : ''}`}> + {item.component} +
+ })} + +
+
+} + +export default Menu diff --git a/demo/src/platform/mobile/rtc/agent/index.module.scss b/demo/src/platform/mobile/rtc/agent/index.module.scss new file mode 100644 index 00000000..fa3ae2ec --- /dev/null +++ b/demo/src/platform/mobile/rtc/agent/index.module.scss @@ -0,0 +1,31 @@ +.agent { + position: relative; + display: flex; + height: 292px; + padding: 20px 16px; + flex-direction: column; + justify-content: flex-start; + align-items: center; + align-self: stretch; + background: linear-gradient(154deg, rgba(27, 66, 166, 0.16) 0%, rgba(27, 45, 140, 0.00) 18%), linear-gradient(153deg, rgba(23, 24, 28, 0.00) 53.75%, #11174E 100%), #0F0F11; + box-shadow: 0px 3.999px 48.988px 0px rgba(0, 7, 72, 0.12); + backdrop-filter: blur(7); + box-sizing: border-box; + + .text { + margin-top: 50px; + color: var(--theme-color, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .view { + margin-top: 32px; + display: flex; + align-items: center; + justify-content: center; + height: 56px; + } +} diff --git a/demo/src/platform/mobile/rtc/agent/index.tsx b/demo/src/platform/mobile/rtc/agent/index.tsx new file mode 100644 index 00000000..a7fd7944 --- /dev/null +++ b/demo/src/platform/mobile/rtc/agent/index.tsx @@ -0,0 +1,34 @@ +"use client" + +import { useAppSelector, useMultibandTrackVolume } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import styles from "./index.module.scss" + +interface AgentProps { + audioTrack?: IMicrophoneAudioTrack +} + +const Agent = (props: AgentProps) => { + const { audioTrack } = props + + const subscribedVolumes = useMultibandTrackVolume(audioTrack, 12); + + return
+
Agent
+
+ +
+
+ +} + + +export default Agent; diff --git a/demo/src/platform/mobile/rtc/audioVisualizer/index.module.scss b/demo/src/platform/mobile/rtc/audioVisualizer/index.module.scss new file mode 100644 index 00000000..1beae944 --- /dev/null +++ b/demo/src/platform/mobile/rtc/audioVisualizer/index.module.scss @@ -0,0 +1,17 @@ +.audioVisualizer { + display: flex; + justify-content: center; + align-items: center; + + + .item {} + + .agent { + background-color: var(--theme-color, #EAECF0); + box-shadow: 0 0 10px var(--theme-color, #EAECF0); + } + + .user { + background-color: var(--Grey-300, #EAECF0); + } +} diff --git a/demo/src/platform/mobile/rtc/audioVisualizer/index.tsx b/demo/src/platform/mobile/rtc/audioVisualizer/index.tsx new file mode 100644 index 00000000..bc21f554 --- /dev/null +++ b/demo/src/platform/mobile/rtc/audioVisualizer/index.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useState, useEffect } from "react" +import styles from "./index.module.scss" + +interface AudioVisualizerProps { + type: "agent" | "user"; + frequencies: Float32Array[]; + gap: number; + barWidth: number; + minBarHeight: number; + maxBarHeight: number + borderRadius: number; +} + + +const AudioVisualizer = (props: AudioVisualizerProps) => { + const { frequencies, gap, barWidth, minBarHeight, maxBarHeight, borderRadius, type } = props; + + const summedFrequencies = frequencies.map((bandFrequencies) => { + const sum = bandFrequencies.reduce((a, b) => a + b, 0) + if (sum <= 0) { + return 0 + } + return Math.sqrt(sum / bandFrequencies.length); + }); + + return
{ + summedFrequencies.map((frequency, index) => { + + const style = { + height: minBarHeight + frequency * (maxBarHeight - minBarHeight) + "px", + borderRadius: borderRadius + "px", + width: barWidth + "px", + transition: + "background-color 0.35s ease-out, transform 0.25s ease-out", + // transform: transform, + } + + return + }) + }
+} + + +export default AudioVisualizer; diff --git a/demo/src/platform/mobile/rtc/camSection/camSelect/index.module.scss b/demo/src/platform/mobile/rtc/camSection/camSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/demo/src/platform/mobile/rtc/camSection/camSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/demo/src/platform/mobile/rtc/camSection/camSelect/index.tsx b/demo/src/platform/mobile/rtc/camSection/camSelect/index.tsx new file mode 100644 index 00000000..33a5e003 --- /dev/null +++ b/demo/src/platform/mobile/rtc/camSection/camSelect/index.tsx @@ -0,0 +1,57 @@ +"use client" + +import AgoraRTC, { ICameraVideoTrack } from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" + +import styles from "./index.module.scss" + +interface CamSelectProps { + videoTrack?: ICameraVideoTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const CamSelect = (props: CamSelectProps) => { + const { videoTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (videoTrack) { + const label = videoTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getCameras().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [videoTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (videoTrack) { + await videoTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default CamSelect diff --git a/demo/src/platform/mobile/rtc/camSection/index.module.scss b/demo/src/platform/mobile/rtc/camSection/index.module.scss new file mode 100644 index 00000000..76f4ad1e --- /dev/null +++ b/demo/src/platform/mobile/rtc/camSection/index.module.scss @@ -0,0 +1,54 @@ +.camera { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .title { + margin-bottom: 10px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 500; + line-height: 150%; + letter-spacing: 0.449px; + } + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + .select { + flex: 0 0 auto; + width: 200px; + } + } + + .view { + position: relative; + margin-top: 12px; + min-height: 210px; + height: 210px; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/demo/src/platform/mobile/rtc/camSection/index.tsx b/demo/src/platform/mobile/rtc/camSection/index.tsx new file mode 100644 index 00000000..2bd0e8db --- /dev/null +++ b/demo/src/platform/mobile/rtc/camSection/index.tsx @@ -0,0 +1,42 @@ +"use client" + +import CamSelect from "./camSelect" +import { CamIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { ICameraVideoTrack } from 'agora-rtc-sdk-ng'; +import { LocalStreamPlayer } from "../streamPlayer" +import { useState, useEffect, useMemo } from 'react'; +import { useSmallScreen } from "@/common" + +interface CamSectionProps { + videoTrack?: ICameraVideoTrack +} + +const CamSection = (props: CamSectionProps) => { + const { videoTrack } = props + const [videoMute, setVideoMute] = useState(false) + + useEffect(() => { + videoTrack?.setMuted(videoMute) + }, [videoTrack, videoMute]) + + const onClickMute = () => { + setVideoMute(!videoMute) + } + + return
+
CAMERA
+
+ + + + +
+
+ +
+
+} + + +export default CamSection; diff --git a/demo/src/platform/mobile/rtc/index.module.scss b/demo/src/platform/mobile/rtc/index.module.scss new file mode 100644 index 00000000..ff7b7958 --- /dev/null +++ b/demo/src/platform/mobile/rtc/index.module.scss @@ -0,0 +1,55 @@ +.rtc { + flex: 0 0 420px; + display: flex; + flex-direction: column; + align-items: flex-start; + flex-shrink: 0; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + box-sizing: border-box; + + .header { + display: flex; + height: 40px; + padding: 0px 16px; + align-items: center; + align-self: stretch; + border-bottom: 1px solid #272A2F; + + .text { + flex: 1 1 auto; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + color: var(--Grey-300, #EAECF0); + } + + .voiceSelect { + flex: 0 0 120px; + } + } + + .you { + display: flex; + padding: 24px 16px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 24px; + align-self: stretch; + border-top: 1px solid #272A2F; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + text-align: center; + } + + + } +} diff --git a/demo/src/platform/mobile/rtc/index.tsx b/demo/src/platform/mobile/rtc/index.tsx new file mode 100644 index 00000000..bc15c070 --- /dev/null +++ b/demo/src/platform/mobile/rtc/index.tsx @@ -0,0 +1,128 @@ +"use client" + +import { ICameraVideoTrack, IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" +import { useAppSelector, useAppDispatch, VOICE_OPTIONS } from "@/common" +import { ITextItem } from "@/types" +import { rtcManager, IUserTracks, IRtcUser } from "@/manager" +import { setRoomConnected, addChatItem, setVoiceType } from "@/store/reducers/global" +import MicSection from "./micSection" +import CamSection from "./camSection" +import Agent from "./agent" +import styles from "./index.module.scss" +import { useRef, useEffect, useState, Fragment } from "react" +import { VoiceIcon } from "@/components/icons" +import CustomSelect from "@/components/customSelect" + +let hasInit = false + +const Rtc = () => { + const dispatch = useAppDispatch() + const options = useAppSelector(state => state.global.options) + const voiceType = useAppSelector(state => state.global.voiceType) + const agentConnected = useAppSelector(state => state.global.agentConnected) + const { userId, channel } = options + const [videoTrack, setVideoTrack] = useState() + const [audioTrack, setAudioTrack] = useState() + const [remoteuser, setRemoteUser] = useState() + + useEffect(() => { + if (!options.channel) { + return + } + if (hasInit) { + return + } + + init() + + return () => { + if (hasInit) { + destory() + } + } + }, [options.channel]) + + + const init = async () => { + console.log("[test] init") + rtcManager.on("localTracksChanged", onLocalTracksChanged) + rtcManager.on("textChanged", onTextChanged) + rtcManager.on("remoteUserChanged", onRemoteUserChanged) + await rtcManager.createTracks() + await rtcManager.join({ + channel, + userId + }) + await rtcManager.publish() + dispatch(setRoomConnected(true)) + hasInit = true + } + + const destory = async () => { + console.log("[test] destory") + rtcManager.off("textChanged", onTextChanged) + rtcManager.off("localTracksChanged", onLocalTracksChanged) + rtcManager.off("remoteUserChanged", onRemoteUserChanged) + await rtcManager.destroy() + dispatch(setRoomConnected(false)) + hasInit = false + } + + const onRemoteUserChanged = (user: IRtcUser) => { + console.log("[test] onRemoteUserChanged", user) + setRemoteUser(user) + } + + const onLocalTracksChanged = (tracks: IUserTracks) => { + console.log("[test] onLocalTracksChanged", tracks) + const { videoTrack, audioTrack } = tracks + if (videoTrack) { + setVideoTrack(videoTrack) + } + if (audioTrack) { + setAudioTrack(audioTrack) + } + } + + const onTextChanged = (text: ITextItem) => { + if (text.dataType == "transcribe") { + const isAgent = Number(text.uid) != Number(userId) + dispatch(addChatItem({ + userId: text.uid, + text: text.text, + type: isAgent ? "agent" : "user", + isFinal: text.isFinal, + time: text.time + })) + } + } + + const onVoiceChange = (value: any) => { + dispatch(setVoiceType(value)) + } + + + return
+
+ Audio & Video + } + options={VOICE_OPTIONS} onChange={onVoiceChange}> +
+ {/* agent */} + + {/* you */} +
+
You
+ {/* microphone */} + + {/* camera */} + +
+
+} + + +export default Rtc; diff --git a/demo/src/platform/mobile/rtc/micSection/index.module.scss b/demo/src/platform/mobile/rtc/micSection/index.module.scss new file mode 100644 index 00000000..60cc6fe1 --- /dev/null +++ b/demo/src/platform/mobile/rtc/micSection/index.module.scss @@ -0,0 +1,58 @@ +.microphone { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .title { + margin-bottom: 10px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 500; + line-height: 150%; + letter-spacing: 0.449px; + } + + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + + } + + .view { + margin-top: 12px; + display: flex; + height: 120px; + padding: 24px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + align-self: stretch; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/demo/src/platform/mobile/rtc/micSection/index.tsx b/demo/src/platform/mobile/rtc/micSection/index.tsx new file mode 100644 index 00000000..3c739159 --- /dev/null +++ b/demo/src/platform/mobile/rtc/micSection/index.tsx @@ -0,0 +1,70 @@ +"use client" + +import { useEffect, useMemo, useState } from "react" +import { useMultibandTrackVolume, useSmallScreen } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { MicIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import MicSelect from "./micSelect"; + +interface MicSectionProps { + audioTrack?: IMicrophoneAudioTrack +} + +const MicSection = (props: MicSectionProps) => { + const { audioTrack } = props + const [audioMute, setAudioMute] = useState(false) + const [mediaStreamTrack, setMediaStreamTrack] = useState() + + + + useEffect(() => { + audioTrack?.on("track-updated", onAudioTrackupdated) + if (audioTrack) { + setMediaStreamTrack(audioTrack.getMediaStreamTrack()) + } + + return () => { + audioTrack?.off("track-updated", onAudioTrackupdated) + } + }, [audioTrack]) + + useEffect(() => { + audioTrack?.setMuted(audioMute) + }, [audioTrack, audioMute]) + + const subscribedVolumes = useMultibandTrackVolume(mediaStreamTrack, 20); + + const onAudioTrackupdated = (track: MediaStreamTrack) => { + console.log("[test] audio track updated", track) + setMediaStreamTrack(track) + } + + const onClickMute = () => { + setAudioMute(!audioMute) + } + + return
+
MICROPHONE
+
+ + + + +
+
+ +
+
+} + + +export default MicSection; diff --git a/demo/src/platform/mobile/rtc/micSection/micSelect/index.module.scss b/demo/src/platform/mobile/rtc/micSection/micSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/demo/src/platform/mobile/rtc/micSection/micSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/demo/src/platform/mobile/rtc/micSection/micSelect/index.tsx b/demo/src/platform/mobile/rtc/micSection/micSelect/index.tsx new file mode 100644 index 00000000..efc842b5 --- /dev/null +++ b/demo/src/platform/mobile/rtc/micSection/micSelect/index.tsx @@ -0,0 +1,58 @@ +"use client" + +import AgoraRTC from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" +import { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" + +import styles from "./index.module.scss" + +interface MicSelectProps { + audioTrack?: IMicrophoneAudioTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const MicSelect = (props: MicSelectProps) => { + const { audioTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (audioTrack) { + const label = audioTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getMicrophones().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [audioTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (audioTrack) { + await audioTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default MicSelect diff --git a/demo/src/platform/mobile/rtc/streamPlayer/index.module.scss b/demo/src/platform/mobile/rtc/streamPlayer/index.module.scss new file mode 100644 index 00000000..b1c57c10 --- /dev/null +++ b/demo/src/platform/mobile/rtc/streamPlayer/index.module.scss @@ -0,0 +1,6 @@ +.streamPlayer { + position: relative; + width: 100%; + height: 100%; + overflow: hidden; +} diff --git a/demo/src/platform/mobile/rtc/streamPlayer/index.tsx b/demo/src/platform/mobile/rtc/streamPlayer/index.tsx new file mode 100644 index 00000000..ba78e377 --- /dev/null +++ b/demo/src/platform/mobile/rtc/streamPlayer/index.tsx @@ -0,0 +1 @@ +export * from "./localStreamPlayer" diff --git a/demo/src/platform/mobile/rtc/streamPlayer/localStreamPlayer.tsx b/demo/src/platform/mobile/rtc/streamPlayer/localStreamPlayer.tsx new file mode 100644 index 00000000..e3e7f06a --- /dev/null +++ b/demo/src/platform/mobile/rtc/streamPlayer/localStreamPlayer.tsx @@ -0,0 +1,46 @@ +"use client" + +import { + ICameraVideoTrack, + IMicrophoneAudioTrack, + IRemoteAudioTrack, + IRemoteVideoTrack, + VideoPlayerConfig, +} from "agora-rtc-sdk-ng" +import { useRef, useState, useLayoutEffect, forwardRef, useEffect, useMemo } from "react" + +import styles from "./index.module.scss" + +interface StreamPlayerProps { + videoTrack?: ICameraVideoTrack + audioTrack?: IMicrophoneAudioTrack + style?: React.CSSProperties + fit?: "cover" | "contain" | "fill" + onClick?: () => void + mute?: boolean +} + +export const LocalStreamPlayer = forwardRef((props: StreamPlayerProps, ref) => { + const { videoTrack, audioTrack, mute = false, style = {}, fit = "cover", onClick = () => { } } = props + const vidDiv = useRef(null) + + useLayoutEffect(() => { + const config = { fit } as VideoPlayerConfig + if (mute) { + videoTrack?.stop() + } else { + if (!videoTrack?.isPlaying) { + videoTrack?.play(vidDiv.current!, config) + } + } + + return () => { + videoTrack?.stop() + } + }, [videoTrack, fit, mute]) + + // local audio track need not to be played + // useLayoutEffect(() => {}, [audioTrack, localAudioMute]) + + return
+}) diff --git a/demo/src/platform/pc/chat/chatItem/index.module.scss b/demo/src/platform/pc/chat/chatItem/index.module.scss new file mode 100644 index 00000000..f28ef7ee --- /dev/null +++ b/demo/src/platform/pc/chat/chatItem/index.module.scss @@ -0,0 +1,90 @@ +.agentChatItem { + width: 100%; + display: flex; + justify-content: flex-start; + + .left { + flex: 0 0 auto; + display: flex; + width: 32px; + height: 32px; + padding: 10px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + border-radius: 200px; + background: var(--Grey-700, #475467); + + .userName { + color: var(---white, #FFF); + text-align: center; + font-size: 14px; + font-weight: 500; + line-height: 150%; + } + } + + .right { + margin-left: 12px; + flex: 1 1 auto; + + .userName { + font-size: 14px; + font-weight: 500; + line-height: 20px; + color: var(--theme-color, #667085) !important; + } + + + .agent { + color: var(--theme-color, #EAECF0) !important; + } + + } +} + +.userChatItem { + width: 100%; + display: flex; + flex-direction: column; + justify-content: flex-end; + align-items: flex-end; + + .userName { + text-align: right; + color: var(--Grey-600, #667085); + font-weight: 500; + line-height: 20px; + } + + + +} + + +.chatItem { + .text { + max-width: 80%; + width: fit-content; + margin-top: 6px; + color: #FFF; + display: flex; + padding: 8px 14px; + flex-direction: column; + justify-content: center; + align-items: flex-start; + font-size: 14px; + font-weight: 400; + line-height: 21px; + white-space: pre-wrap; + border-radius: 0px 8px 8px 8px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} + +.chatItem+.chatItem { + margin-top: 14px; +} diff --git a/demo/src/platform/pc/chat/chatItem/index.tsx b/demo/src/platform/pc/chat/chatItem/index.tsx new file mode 100644 index 00000000..6364aaea --- /dev/null +++ b/demo/src/platform/pc/chat/chatItem/index.tsx @@ -0,0 +1,51 @@ +import { IChatItem } from "@/types" +import styles from "./index.module.scss" +import { usePrevious } from "@/common" +import { use, useEffect, useMemo, useState } from "react" + +interface ChatItemProps { + data: IChatItem +} + + +const AgentChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + return
+ + Ag + + +
Agent
+
+ {text} +
+ +
+} + +const UserChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + return
+
You
+
{text}
+
+} + + +const ChatItem = (props: ChatItemProps) => { + const { data } = props + + + return ( + data.type === "agent" ? : + ); + + +} + + +export default ChatItem diff --git a/demo/src/platform/pc/chat/index.module.scss b/demo/src/platform/pc/chat/index.module.scss new file mode 100644 index 00000000..39c4956d --- /dev/null +++ b/demo/src/platform/pc/chat/index.module.scss @@ -0,0 +1,79 @@ +.chat { + flex: 1 1 auto; + min-width: 500px; + display: flex; + flex-direction: column; + align-items: flex-start; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + overflow: hidden; + + .header { + display: flex; + height: 42px; + padding: 0px 16px; + align-items: center; + align-self: stretch; + border-bottom: 1px solid #272A2F; + + .left { + flex: 1 1 auto; + display: flex; + align-items: center; + gap: 5px; + + .text { + margin-left: 4px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + height: 40px; + line-height: 40px; + letter-spacing: 0.449px; + } + + .languageSelect { + width: 100px; + } + } + + + .right { + display: flex; + align-items: center; + gap: 10px; + flex: 0 0 230px; + justify-content: right; + } + + } + + .content { + display: flex; + padding: 12px 24px; + flex-direction: column; + align-items: flex-start; + gap: 10px; + flex: 1 0 500px; + align-self: stretch; + overflow-y: auto; + + + &::-webkit-scrollbar { + width: 6px + } + + &::-webkit-scrollbar-track { + background-color: transparent; + } + + &::-webkit-scrollbar-thumb { + background-color: #6B6B6B; + border-radius: 4px; + } + } + + +} diff --git a/demo/src/platform/pc/chat/index.tsx b/demo/src/platform/pc/chat/index.tsx new file mode 100644 index 00000000..64dea171 --- /dev/null +++ b/demo/src/platform/pc/chat/index.tsx @@ -0,0 +1,66 @@ +"use client" + +import { ReactElement, useEffect, useRef, useState } from "react" +import ChatItem from "./chatItem" +import { + genRandomChatList, useAppDispatch, useAutoScroll, + LANGUAGE_OPTIONS, useAppSelector, + GRAPH_OPTIONS, + isRagGraph, +} from "@/common" +import { setGraphName, setLanguage } from "@/store/reducers/global" +import { Select, } from 'antd'; +import PdfSelect from "@/components/pdfSelect" + +import styles from "./index.module.scss" + + + + +const Chat = () => { + const dispatch = useAppDispatch() + const chatItems = useAppSelector(state => state.global.chatItems) + const language = useAppSelector(state => state.global.language) + const graphName = useAppSelector(state => state.global.graphName) + const agentConnected = useAppSelector(state => state.global.agentConnected) + + // const chatItems = genRandomChatList(10) + const chatRef = useRef(null) + + + useAutoScroll(chatRef) + + + const onLanguageChange = (val: any) => { + dispatch(setLanguage(val)) + } + + const onGraphNameChange = (val: any) => { + dispatch(setGraphName(val)) + } + + + return
+
+ + + + + + {isRagGraph(graphName) ? : null} + +
+
+ {chatItems.map((item, index) => { + return + })} +
+
+} + + +export default Chat diff --git a/demo/src/platform/pc/description/index.module.scss b/demo/src/platform/pc/description/index.module.scss new file mode 100644 index 00000000..50b29301 --- /dev/null +++ b/demo/src/platform/pc/description/index.module.scss @@ -0,0 +1,73 @@ +.description { + position: relative; + display: flex; + padding: 12px 16px; + align-items: center; + gap: 12px; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-style: normal; + font-weight: 600; + /* 21px */ + letter-spacing: 0.449px; + } + + .text { + margin-left: 12px; + flex: 1 1 auto; + color: var(--Grey-600, #667085); + font-size: 14px; + font-style: normal; + font-weight: 400; + } + + + .btnConnect { + width: 150px; + display: flex; + padding: 8px 14px; + justify-content: center; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 6px; + background: var(--theme-color, #0888FF); + border: 1px solid var(--theme-color, #0888FF); + box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05); + cursor: pointer; + user-select: none; + caret-color: transparent; + box-sizing: border-box; + + .btnText { + width: 100px; + text-align: center; + color: var(---White, #FFF); + font-size: 14px; + font-weight: 500; + line-height: 20px; + } + + .btnText.disconnect { + color: var(--Error-400-T, #E95C7B); + } + + + .loading { + margin-left: 4px; + } + } + + + .btnConnect.disconnect { + background: #181A1D; + border: 1px solid var(--Error-400-T, #E95C7B); + } + +} diff --git a/demo/src/platform/pc/description/index.tsx b/demo/src/platform/pc/description/index.tsx new file mode 100644 index 00000000..a9a055cd --- /dev/null +++ b/demo/src/platform/pc/description/index.tsx @@ -0,0 +1,101 @@ +import { setAgentConnected } from "@/store/reducers/global" +import { + DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, + apiStartService, apiStopService +} from "@/common" +import { Select, Button, message, Upload } from "antd" +import { useEffect, useState, MouseEventHandler } from "react" +import { LoadingOutlined, UploadOutlined } from "@ant-design/icons" +import styles from "./index.module.scss" + +let intervalId: any + +const Description = () => { + const dispatch = useAppDispatch() + const agentConnected = useAppSelector(state => state.global.agentConnected) + const channel = useAppSelector(state => state.global.options.channel) + const userId = useAppSelector(state => state.global.options.userId) + const language = useAppSelector(state => state.global.language) + const voiceType = useAppSelector(state => state.global.voiceType) + const graphName = useAppSelector(state => state.global.graphName) + const [loading, setLoading] = useState(false) + + useEffect(() => { + if (channel) { + checkAgentConnected() + } + }, [channel]) + + + const checkAgentConnected = async () => { + const res: any = await apiPing(channel) + if (res?.code == 0) { + dispatch(setAgentConnected(true)) + } + } + + const onClickConnect = async () => { + if (loading) { + return + } + setLoading(true) + if (agentConnected) { + await apiStopService(channel) + dispatch(setAgentConnected(false)) + message.success("Agent disconnected") + stopPing() + } else { + const res = await apiStartService({ + channel, + userId, + graphName, + language, + voiceType + }) + const { code, msg } = res || {} + if (code != 0) { + if (code == "10001") { + message.error("The number of users experiencing the program simultaneously has exceeded the limit. Please try again later.") + } else { + message.error(`code:${code},msg:${msg}`) + } + setLoading(false) + throw new Error(msg) + } + dispatch(setAgentConnected(true)) + message.success("Agent connected") + startPing() + } + setLoading(false) + } + + const startPing = () => { + if (intervalId) { + stopPing() + } + intervalId = setInterval(() => { + apiPing(channel) + }, 3000) + } + + const stopPing = () => { + if (intervalId) { + clearInterval(intervalId) + intervalId = null + } + } + + return
+ Description + Astra is a multimodal agent powered by TEN + + + {!agentConnected ? "Connect" : "Disconnect"} + {loading ? : null} + + +
+} + + +export default Description diff --git a/demo/src/platform/pc/entry/index.module.scss b/demo/src/platform/pc/entry/index.module.scss new file mode 100644 index 00000000..5f161d2c --- /dev/null +++ b/demo/src/platform/pc/entry/index.module.scss @@ -0,0 +1,27 @@ +.entry { + position: relative; + height: 100%; + box-sizing: border-box; + display: flex; + flex-direction: column; + + .content { + position: relative; + padding: 16px; + box-sizing: border-box; + display: flex; + height: calc(100% - 64px); + flex-direction: column; + + .body { + margin-top: 16px; + display: flex; + gap: 24px; + flex-grow: 1; + .chat { + display: flex; + flex-grow: 1; + } + } + } +} diff --git a/demo/src/platform/pc/entry/index.tsx b/demo/src/platform/pc/entry/index.tsx new file mode 100644 index 00000000..a7ee7592 --- /dev/null +++ b/demo/src/platform/pc/entry/index.tsx @@ -0,0 +1,26 @@ +import Chat from "../chat" +import Description from "../description" +import Rtc from "../rtc" +import Header from "../header" + +import styles from "./index.module.scss" + +const PCEntry = () => { + return
+
+
+ +
+
+ +
+
+ +
+
+
+
+} + + +export default PCEntry diff --git a/demo/src/platform/pc/header/index.module.scss b/demo/src/platform/pc/header/index.module.scss new file mode 100644 index 00000000..31138cc7 --- /dev/null +++ b/demo/src/platform/pc/header/index.module.scss @@ -0,0 +1,58 @@ +.header { + display: flex; + width: 100%; + height: 48px; + padding: 24px; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #24262A; + background: #1E2024; + box-shadow: 0px 12px 16px -4px rgba(8, 15, 52, 0.06), 0px 4px 6px -2px rgba(8, 15, 52, 0.03); + box-sizing: border-box; + z-index: 999; + + .logoWrapper { + display: flex; + align-items: center; + + .text { + margin-left: 8px; + color: var(---white, #FFF); + text-align: right; + font-family: Inter; + font-size: 16px; + font-weight: 500; + } + } + + .content { + display: flex; + align-items: center; + justify-content: center; + height: 48px; + flex: 1 1 auto; + color: var(--Grey-300, #EAECF0); + font-size: 16px; + font-weight: 500; + line-height: 48px; + letter-spacing: 0.449px; + text-align: center; + + .text { + margin-left: 4px; + } + } + + .links { + display: flex; + align-items: center; + gap: 8px; + + span { + display: flex; + } + } + .githubWrapper { + cursor: pointer; + } +} diff --git a/demo/src/platform/pc/header/index.tsx b/demo/src/platform/pc/header/index.tsx new file mode 100644 index 00000000..a55b3f04 --- /dev/null +++ b/demo/src/platform/pc/header/index.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useAppSelector, GITHUB_URL, useSmallScreen } from "@/common" +import Network from "./network" +import InfoPopover from "./infoPopover" +import StylePopover from "./stylePopover" +import { GithubIcon, LogoIcon, InfoIcon, ColorPickerIcon } from "@/components/icons" + +import styles from "./index.module.scss" + +const Header = () => { + const themeColor = useAppSelector(state => state.global.themeColor) + const options = useAppSelector(state => state.global.options) + const { channel } = options + + + const onClickGithub = () => { + if (typeof window !== "undefined") { + window.open(GITHUB_URL, "_blank") + } + } + + + + return
+ + + + + + + Channel Name: {channel} + + +
+ + + + + + + +
+
+} + + +export default Header diff --git a/demo/src/platform/pc/header/infoPopover/index.module.scss b/demo/src/platform/pc/header/infoPopover/index.module.scss new file mode 100644 index 00000000..cd3f72f8 --- /dev/null +++ b/demo/src/platform/pc/header/infoPopover/index.module.scss @@ -0,0 +1,43 @@ +.info { + display: flex; + padding: 12px 16px; + flex-direction: column; + align-items: flex-start; + gap: 8px; + align-self: stretch; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .item { + width: 100%; + display: flex; + justify-content: space-between; + align-items: center; + + .title { + color: var(--Grey-600, #667085); + font-size: 14px; + font-weight: 400; + line-height: 150%; + } + + .content { + color: var(--theme-color, #FFF); + font-size: 14px; + font-weight: 400; + line-height: 150%; + } + } + + .slider { + height: 1px; + width: 100%; + background-color: #0D0F12; + } +} diff --git a/demo/src/platform/pc/header/infoPopover/index.tsx b/demo/src/platform/pc/header/infoPopover/index.tsx new file mode 100644 index 00000000..cd451418 --- /dev/null +++ b/demo/src/platform/pc/header/infoPopover/index.tsx @@ -0,0 +1,57 @@ +import { useMemo } from "react" +import { useAppSelector } from "@/common" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface InfoPopoverProps { + children?: React.ReactNode +} + +const InfoPopover = (props: InfoPopoverProps) => { + const { children } = props + const options = useAppSelector(state => state.global.options) + const { channel, userId } = options + + const roomConnected = useAppSelector(state => state.global.roomConnected) + const agentConnected = useAppSelector(state => state.global.agentConnected) + + const roomConnectedText = useMemo(() => { + return roomConnected ? "TRUE" : "FALSE" + }, [roomConnected]) + + const agentConnectedText = useMemo(() => { + return agentConnected ? "TRUE" : "FALSE" + }, [agentConnected]) + + + + const content =
+
INFO
+
+ Room + {channel} +
+
+ Participant + {userId} +
+
+
STATUS
+
+
Room connected
+
{roomConnectedText}
+
+
+
Agent connected
+
{agentConnectedText}
+
+
+ + + return {children} + +} + +export default InfoPopover diff --git a/demo/src/platform/pc/header/network/index.module.scss b/demo/src/platform/pc/header/network/index.module.scss new file mode 100644 index 00000000..e69de29b diff --git a/demo/src/platform/pc/header/network/index.tsx b/demo/src/platform/pc/header/network/index.tsx new file mode 100644 index 00000000..92b4e33b --- /dev/null +++ b/demo/src/platform/pc/header/network/index.tsx @@ -0,0 +1,37 @@ +"use client"; + +import React from "react"; +import { rtcManager } from "@/manager" +import { NetworkQuality } from "agora-rtc-sdk-ng" +import { useEffect, useState } from "react" +import { NetworkIcon } from "@/components/icons" + +interface NetworkProps { + style?: React.CSSProperties +} + +const NetWork = (props: NetworkProps) => { + const { style } = props + + const [networkQuality, setNetworkQuality] = useState() + + useEffect(() => { + rtcManager.on("networkQuality", onNetworkQuality) + + return () => { + rtcManager.off("networkQuality", onNetworkQuality) + } + }, []) + + const onNetworkQuality = (quality: NetworkQuality) => { + setNetworkQuality(quality) + } + + return ( + + + + ) +} + +export default NetWork diff --git a/demo/src/platform/pc/header/stylePopover/colorPicker/index.module.scss b/demo/src/platform/pc/header/stylePopover/colorPicker/index.module.scss new file mode 100644 index 00000000..405e7781 --- /dev/null +++ b/demo/src/platform/pc/header/stylePopover/colorPicker/index.module.scss @@ -0,0 +1,24 @@ +.colorPicker { + height: 24px; + display: flex; + align-items: center; + + :global(.react-colorful) { + width: 220px; + height: 8px; + } + + :global(.react-colorful__saturation) { + display: none; + } + + :global(.react-colorful__hue) { + border-radius: 8px !important; + height: 8px; + } + + :global(.react-colorful__pointer) { + width: 24px; + height: 24px; + } +} diff --git a/demo/src/platform/pc/header/stylePopover/colorPicker/index.tsx b/demo/src/platform/pc/header/stylePopover/colorPicker/index.tsx new file mode 100644 index 00000000..28163d77 --- /dev/null +++ b/demo/src/platform/pc/header/stylePopover/colorPicker/index.tsx @@ -0,0 +1,22 @@ +"use client" + +import { HexColorPicker } from "react-colorful"; +import { useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import styles from "./index.module.scss"; + +const ColorPicker = () => { + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + const onColorChange = (color: string) => { + console.log(color); + dispatch(setThemeColor(color)) + }; + + return
+ +
+}; + +export default ColorPicker; diff --git a/demo/src/platform/pc/header/stylePopover/index.module.scss b/demo/src/platform/pc/header/stylePopover/index.module.scss new file mode 100644 index 00000000..98c7f182 --- /dev/null +++ b/demo/src/platform/pc/header/stylePopover/index.module.scss @@ -0,0 +1,51 @@ +.info { + display: flex; + padding: 12px 16px; + flex-direction: column; + align-items: flex-start; + gap: 16px; + align-self: stretch; + + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .color { + font-size: 0; + white-space: nowrap; + + .item { + position: relative; + display: inline-block; + width: 28px; + height: 28px; + border-radius: 4px; + border: 2px solid transparent; + font-size: 0; + cursor: pointer; + + .inner { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + width: 18px; + height: 18px; + border-radius: 2px; + box-sizing: border-box; + } + } + + .item+.item { + margin-left: 12px; + } + + } + + +} diff --git a/demo/src/platform/pc/header/stylePopover/index.tsx b/demo/src/platform/pc/header/stylePopover/index.tsx new file mode 100644 index 00000000..f8508323 --- /dev/null +++ b/demo/src/platform/pc/header/stylePopover/index.tsx @@ -0,0 +1,54 @@ +import { useMemo } from "react" +import { COLOR_LIST, useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import ColorPicker from "./colorPicker" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface StylePopoverProps { + children?: React.ReactNode +} + +const StylePopover = (props: StylePopoverProps) => { + const { children } = props + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + + const onClickColor = (index: number) => { + const target = COLOR_LIST[index] + if (target.active !== themeColor) { + dispatch(setThemeColor(target.active)) + } + } + + const content =
+
STYLE
+
+ { + COLOR_LIST.map((item, index) => { + return onClickColor(index)} + className={styles.item} + key={index}> + + + }) + } +
+ +
+ + + return {children} + +} + +export default StylePopover diff --git a/demo/src/platform/pc/rtc/agent/index.module.scss b/demo/src/platform/pc/rtc/agent/index.module.scss new file mode 100644 index 00000000..fa3ae2ec --- /dev/null +++ b/demo/src/platform/pc/rtc/agent/index.module.scss @@ -0,0 +1,31 @@ +.agent { + position: relative; + display: flex; + height: 292px; + padding: 20px 16px; + flex-direction: column; + justify-content: flex-start; + align-items: center; + align-self: stretch; + background: linear-gradient(154deg, rgba(27, 66, 166, 0.16) 0%, rgba(27, 45, 140, 0.00) 18%), linear-gradient(153deg, rgba(23, 24, 28, 0.00) 53.75%, #11174E 100%), #0F0F11; + box-shadow: 0px 3.999px 48.988px 0px rgba(0, 7, 72, 0.12); + backdrop-filter: blur(7); + box-sizing: border-box; + + .text { + margin-top: 50px; + color: var(--theme-color, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .view { + margin-top: 32px; + display: flex; + align-items: center; + justify-content: center; + height: 56px; + } +} diff --git a/demo/src/platform/pc/rtc/agent/index.tsx b/demo/src/platform/pc/rtc/agent/index.tsx new file mode 100644 index 00000000..a7fd7944 --- /dev/null +++ b/demo/src/platform/pc/rtc/agent/index.tsx @@ -0,0 +1,34 @@ +"use client" + +import { useAppSelector, useMultibandTrackVolume } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import styles from "./index.module.scss" + +interface AgentProps { + audioTrack?: IMicrophoneAudioTrack +} + +const Agent = (props: AgentProps) => { + const { audioTrack } = props + + const subscribedVolumes = useMultibandTrackVolume(audioTrack, 12); + + return
+
Agent
+
+ +
+
+ +} + + +export default Agent; diff --git a/demo/src/platform/pc/rtc/audioVisualizer/index.module.scss b/demo/src/platform/pc/rtc/audioVisualizer/index.module.scss new file mode 100644 index 00000000..1beae944 --- /dev/null +++ b/demo/src/platform/pc/rtc/audioVisualizer/index.module.scss @@ -0,0 +1,17 @@ +.audioVisualizer { + display: flex; + justify-content: center; + align-items: center; + + + .item {} + + .agent { + background-color: var(--theme-color, #EAECF0); + box-shadow: 0 0 10px var(--theme-color, #EAECF0); + } + + .user { + background-color: var(--Grey-300, #EAECF0); + } +} diff --git a/demo/src/platform/pc/rtc/audioVisualizer/index.tsx b/demo/src/platform/pc/rtc/audioVisualizer/index.tsx new file mode 100644 index 00000000..bc21f554 --- /dev/null +++ b/demo/src/platform/pc/rtc/audioVisualizer/index.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useState, useEffect } from "react" +import styles from "./index.module.scss" + +interface AudioVisualizerProps { + type: "agent" | "user"; + frequencies: Float32Array[]; + gap: number; + barWidth: number; + minBarHeight: number; + maxBarHeight: number + borderRadius: number; +} + + +const AudioVisualizer = (props: AudioVisualizerProps) => { + const { frequencies, gap, barWidth, minBarHeight, maxBarHeight, borderRadius, type } = props; + + const summedFrequencies = frequencies.map((bandFrequencies) => { + const sum = bandFrequencies.reduce((a, b) => a + b, 0) + if (sum <= 0) { + return 0 + } + return Math.sqrt(sum / bandFrequencies.length); + }); + + return
{ + summedFrequencies.map((frequency, index) => { + + const style = { + height: minBarHeight + frequency * (maxBarHeight - minBarHeight) + "px", + borderRadius: borderRadius + "px", + width: barWidth + "px", + transition: + "background-color 0.35s ease-out, transform 0.25s ease-out", + // transform: transform, + } + + return + }) + }
+} + + +export default AudioVisualizer; diff --git a/demo/src/platform/pc/rtc/camSection/camSelect/index.module.scss b/demo/src/platform/pc/rtc/camSection/camSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/demo/src/platform/pc/rtc/camSection/camSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/demo/src/platform/pc/rtc/camSection/camSelect/index.tsx b/demo/src/platform/pc/rtc/camSection/camSelect/index.tsx new file mode 100644 index 00000000..33a5e003 --- /dev/null +++ b/demo/src/platform/pc/rtc/camSection/camSelect/index.tsx @@ -0,0 +1,57 @@ +"use client" + +import AgoraRTC, { ICameraVideoTrack } from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" + +import styles from "./index.module.scss" + +interface CamSelectProps { + videoTrack?: ICameraVideoTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const CamSelect = (props: CamSelectProps) => { + const { videoTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (videoTrack) { + const label = videoTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getCameras().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [videoTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (videoTrack) { + await videoTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default CamSelect diff --git a/demo/src/platform/pc/rtc/camSection/index.module.scss b/demo/src/platform/pc/rtc/camSection/index.module.scss new file mode 100644 index 00000000..28b88e2e --- /dev/null +++ b/demo/src/platform/pc/rtc/camSection/index.module.scss @@ -0,0 +1,54 @@ +.camera { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + .text { + flex: 1 1 auto; + height: 32px; + line-height: 32px; + color: var(--Grey-300, #EAECF0); + font-weight: 500; + letter-spacing: 0.449px; + } + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + .select { + flex: 0 0 auto; + width: 200px; + } + } + + .view { + position: relative; + margin-top: 12px; + min-height: 210px; + height: 210px; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/demo/src/platform/pc/rtc/camSection/index.tsx b/demo/src/platform/pc/rtc/camSection/index.tsx new file mode 100644 index 00000000..99e5392c --- /dev/null +++ b/demo/src/platform/pc/rtc/camSection/index.tsx @@ -0,0 +1,47 @@ +"use client" + +import CamSelect from "./camSelect" +import { CamIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { ICameraVideoTrack } from 'agora-rtc-sdk-ng'; +import { LocalStreamPlayer } from "../streamPlayer" +import { useState, useEffect, useMemo } from 'react'; +import { useSmallScreen } from "@/common" + +interface CamSectionProps { + videoTrack?: ICameraVideoTrack +} + +const CamSection = (props: CamSectionProps) => { + const { videoTrack } = props + const [videoMute, setVideoMute] = useState(false) + const { xs } = useSmallScreen() + + const CamText = useMemo(() => { + return xs ? "CAM" : "CAMERA" + }, [xs]) + + useEffect(() => { + videoTrack?.setMuted(videoMute) + }, [videoTrack, videoMute]) + + const onClickMute = () => { + setVideoMute(!videoMute) + } + + return
+
+ {CamText} + + + + +
+
+ +
+
+} + + +export default CamSection; diff --git a/demo/src/platform/pc/rtc/index.module.scss b/demo/src/platform/pc/rtc/index.module.scss new file mode 100644 index 00000000..b62025c5 --- /dev/null +++ b/demo/src/platform/pc/rtc/index.module.scss @@ -0,0 +1,55 @@ +.rtc { + flex: 0 0 420px; + display: flex; + flex-direction: column; + align-items: flex-start; + flex-shrink: 0; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + box-sizing: border-box; + + .header { + display: flex; + height: 42px; + padding: 0px 16px; + align-items: center; + align-self: stretch; + border-bottom: 1px solid #272A2F; + + .text { + flex: 1 1 auto; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + color: var(--Grey-300, #EAECF0); + } + + .voiceSelect { + flex: 0 0 120px; + } + } + + .you { + display: flex; + padding: 24px 16px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 24px; + align-self: stretch; + border-top: 1px solid #272A2F; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + text-align: center; + } + + + } +} diff --git a/demo/src/platform/pc/rtc/index.tsx b/demo/src/platform/pc/rtc/index.tsx new file mode 100644 index 00000000..1195ca0f --- /dev/null +++ b/demo/src/platform/pc/rtc/index.tsx @@ -0,0 +1,128 @@ +"use client" + +import { ICameraVideoTrack, IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" +import { useAppSelector, useAppDispatch, VOICE_OPTIONS } from "@/common" +import { ITextItem } from "@/types" +import { rtcManager, IUserTracks, IRtcUser } from "@/manager" +import { setRoomConnected, addChatItem, setVoiceType } from "@/store/reducers/global" +import MicSection from "./micSection" +import CamSection from "./camSection" +import Agent from "./agent" +import styles from "./index.module.scss" +import { useRef, useEffect, useState, Fragment } from "react" +import { VoiceIcon } from "@/components/icons" +import CustomSelect from "@/components/customSelect" + +let hasInit = false + +const Rtc = () => { + const dispatch = useAppDispatch() + const options = useAppSelector(state => state.global.options) + const voiceType = useAppSelector(state => state.global.voiceType) + const agentConnected = useAppSelector(state => state.global.agentConnected) + const { userId, channel } = options + const [videoTrack, setVideoTrack] = useState() + const [audioTrack, setAudioTrack] = useState() + const [remoteuser, setRemoteUser] = useState() + + useEffect(() => { + if (!options.channel) { + return + } + if (hasInit) { + return + } + + init() + + return () => { + if (hasInit) { + destory() + } + } + }, [options.channel]) + + + const init = async () => { + console.log("[test] init") + rtcManager.on("localTracksChanged", onLocalTracksChanged) + rtcManager.on("textChanged", onTextChanged) + rtcManager.on("remoteUserChanged", onRemoteUserChanged) + await rtcManager.createTracks() + await rtcManager.join({ + channel, + userId + }) + await rtcManager.publish() + dispatch(setRoomConnected(true)) + hasInit = true + } + + const destory = async () => { + console.log("[test] destory") + rtcManager.off("textChanged", onTextChanged) + rtcManager.off("localTracksChanged", onLocalTracksChanged) + rtcManager.off("remoteUserChanged", onRemoteUserChanged) + await rtcManager.destroy() + dispatch(setRoomConnected(false)) + hasInit = false + } + + const onRemoteUserChanged = (user: IRtcUser) => { + console.log("[test] onRemoteUserChanged", user) + setRemoteUser(user) + } + + const onLocalTracksChanged = (tracks: IUserTracks) => { + console.log("[test] onLocalTracksChanged", tracks) + const { videoTrack, audioTrack } = tracks + if (videoTrack) { + setVideoTrack(videoTrack) + } + if (audioTrack) { + setAudioTrack(audioTrack) + } + } + + const onTextChanged = (text: ITextItem) => { + if (text.dataType == "transcribe") { + const isAgent = Number(text.uid) != Number(userId) + dispatch(addChatItem({ + userId: text.uid, + text: text.text, + type: isAgent ? "agent" : "user", + isFinal: text.isFinal, + time: text.time + })) + } + } + + const onVoiceChange = (value: any) => { + dispatch(setVoiceType(value)) + } + + + return
+
+ Audio & Video + } + options={VOICE_OPTIONS} onChange={onVoiceChange}> +
+ {/* agent */} + + {/* you */} +
+
You
+ {/* microphone */} + + {/* camera */} + +
+
+} + + +export default Rtc; diff --git a/demo/src/platform/pc/rtc/micSection/index.module.scss b/demo/src/platform/pc/rtc/micSection/index.module.scss new file mode 100644 index 00000000..81fffd3d --- /dev/null +++ b/demo/src/platform/pc/rtc/micSection/index.module.scss @@ -0,0 +1,56 @@ +.microphone { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + .text { + flex: 1 1 auto; + height: 32px; + line-height: 32px; + color: var(--Grey-300, #EAECF0); + font-weight: 500; + letter-spacing: 0.449px; + } + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + + } + + .view { + margin-top: 12px; + display: flex; + height: 120px; + padding: 24px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + align-self: stretch; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/demo/src/platform/pc/rtc/micSection/index.tsx b/demo/src/platform/pc/rtc/micSection/index.tsx new file mode 100644 index 00000000..6d97f3e2 --- /dev/null +++ b/demo/src/platform/pc/rtc/micSection/index.tsx @@ -0,0 +1,73 @@ +"use client" + +import { useEffect, useMemo, useState } from "react" +import { useMultibandTrackVolume, useSmallScreen } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { MicIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import MicSelect from "./micSelect"; + +interface MicSectionProps { + audioTrack?: IMicrophoneAudioTrack +} + +const MicSection = (props: MicSectionProps) => { + const { audioTrack } = props + const [audioMute, setAudioMute] = useState(false) + const [mediaStreamTrack, setMediaStreamTrack] = useState() + const { xs } = useSmallScreen() + + const MicText = useMemo(() => { + return xs ? "MIC" : "MICROPHONE" + }, [xs]) + + useEffect(() => { + audioTrack?.on("track-updated", onAudioTrackupdated) + if (audioTrack) { + setMediaStreamTrack(audioTrack.getMediaStreamTrack()) + } + + return () => { + audioTrack?.off("track-updated", onAudioTrackupdated) + } + }, [audioTrack]) + + useEffect(() => { + audioTrack?.setMuted(audioMute) + }, [audioTrack, audioMute]) + + const subscribedVolumes = useMultibandTrackVolume(mediaStreamTrack, 20); + + const onAudioTrackupdated = (track: MediaStreamTrack) => { + console.log("[test] audio track updated", track) + setMediaStreamTrack(track) + } + + const onClickMute = () => { + setAudioMute(!audioMute) + } + + return
+
+ {MicText} + + + + +
+
+ +
+
+} + + +export default MicSection; diff --git a/demo/src/platform/pc/rtc/micSection/micSelect/index.module.scss b/demo/src/platform/pc/rtc/micSection/micSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/demo/src/platform/pc/rtc/micSection/micSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/demo/src/platform/pc/rtc/micSection/micSelect/index.tsx b/demo/src/platform/pc/rtc/micSection/micSelect/index.tsx new file mode 100644 index 00000000..efc842b5 --- /dev/null +++ b/demo/src/platform/pc/rtc/micSection/micSelect/index.tsx @@ -0,0 +1,58 @@ +"use client" + +import AgoraRTC from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" +import { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" + +import styles from "./index.module.scss" + +interface MicSelectProps { + audioTrack?: IMicrophoneAudioTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const MicSelect = (props: MicSelectProps) => { + const { audioTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (audioTrack) { + const label = audioTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getMicrophones().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [audioTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (audioTrack) { + await audioTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default MicSelect diff --git a/demo/src/platform/pc/rtc/streamPlayer/index.module.scss b/demo/src/platform/pc/rtc/streamPlayer/index.module.scss new file mode 100644 index 00000000..b1c57c10 --- /dev/null +++ b/demo/src/platform/pc/rtc/streamPlayer/index.module.scss @@ -0,0 +1,6 @@ +.streamPlayer { + position: relative; + width: 100%; + height: 100%; + overflow: hidden; +} diff --git a/demo/src/platform/pc/rtc/streamPlayer/index.tsx b/demo/src/platform/pc/rtc/streamPlayer/index.tsx new file mode 100644 index 00000000..ba78e377 --- /dev/null +++ b/demo/src/platform/pc/rtc/streamPlayer/index.tsx @@ -0,0 +1 @@ +export * from "./localStreamPlayer" diff --git a/demo/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx b/demo/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx new file mode 100644 index 00000000..e3e7f06a --- /dev/null +++ b/demo/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx @@ -0,0 +1,46 @@ +"use client" + +import { + ICameraVideoTrack, + IMicrophoneAudioTrack, + IRemoteAudioTrack, + IRemoteVideoTrack, + VideoPlayerConfig, +} from "agora-rtc-sdk-ng" +import { useRef, useState, useLayoutEffect, forwardRef, useEffect, useMemo } from "react" + +import styles from "./index.module.scss" + +interface StreamPlayerProps { + videoTrack?: ICameraVideoTrack + audioTrack?: IMicrophoneAudioTrack + style?: React.CSSProperties + fit?: "cover" | "contain" | "fill" + onClick?: () => void + mute?: boolean +} + +export const LocalStreamPlayer = forwardRef((props: StreamPlayerProps, ref) => { + const { videoTrack, audioTrack, mute = false, style = {}, fit = "cover", onClick = () => { } } = props + const vidDiv = useRef(null) + + useLayoutEffect(() => { + const config = { fit } as VideoPlayerConfig + if (mute) { + videoTrack?.stop() + } else { + if (!videoTrack?.isPlaying) { + videoTrack?.play(vidDiv.current!, config) + } + } + + return () => { + videoTrack?.stop() + } + }, [videoTrack, fit, mute]) + + // local audio track need not to be played + // useLayoutEffect(() => {}, [audioTrack, localAudioMute]) + + return
+}) diff --git a/demo/src/protobuf/SttMessage.js b/demo/src/protobuf/SttMessage.js new file mode 100644 index 00000000..e69de29b diff --git a/demo/src/protobuf/SttMessage.proto b/demo/src/protobuf/SttMessage.proto new file mode 100644 index 00000000..d8e07a54 --- /dev/null +++ b/demo/src/protobuf/SttMessage.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package Agora.SpeechToText; + +option objc_class_prefix = "Stt"; + +option csharp_namespace = "AgoraSTTSample.Protobuf"; + +option java_package = "io.agora.rtc.speech2text"; +option java_outer_classname = "AgoraSpeech2TextProtobuffer"; + +message Text { + int32 vendor = 1; + int32 version = 2; + int32 seqnum = 3; + int64 uid = 4; + int32 flag = 5; + int64 time = 6; + int32 lang = 7; + int32 starttime = 8; + int32 offtime = 9; + repeated Word words = 10; + bool end_of_segment = 11; + int32 duration_ms = 12; + string data_type = 13; // transcribe ,translate + repeated Translation trans = 14; + string culture = 15; +} +message Word { + string text = 1; + int32 start_ms = 2; + int32 duration_ms = 3; + bool is_final = 4; + double confidence = 5; +} +message Translation { + bool is_final = 1; + string lang = 2; // 翻译语言 + repeated string texts = 3; +} diff --git a/demo/src/protobuf/SttMessage_es6.js b/demo/src/protobuf/SttMessage_es6.js new file mode 100644 index 00000000..54188af8 --- /dev/null +++ b/demo/src/protobuf/SttMessage_es6.js @@ -0,0 +1,134 @@ +/* eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars */ +import * as $protobuf from "protobufjs/light" + +const $root = ($protobuf.roots.default || ($protobuf.roots.default = new $protobuf.Root())).addJSON( + { + Agora: { + nested: { + SpeechToText: { + options: { + objc_class_prefix: "Stt", + csharp_namespace: "AgoraSTTSample.Protobuf", + java_package: "io.agora.rtc.speech2text", + java_outer_classname: "AgoraSpeech2TextProtobuffer", + }, + nested: { + Text: { + fields: { + vendor: { + type: "int32", + id: 1, + }, + version: { + type: "int32", + id: 2, + }, + seqnum: { + type: "int32", + id: 3, + }, + uid: { + type: "uint32", + id: 4, + }, + flag: { + type: "int32", + id: 5, + }, + time: { + type: "int64", + id: 6, + }, + lang: { + type: "int32", + id: 7, + }, + starttime: { + type: "int32", + id: 8, + }, + offtime: { + type: "int32", + id: 9, + }, + words: { + rule: "repeated", + type: "Word", + id: 10, + }, + endOfSegment: { + type: "bool", + id: 11, + }, + durationMs: { + type: "int32", + id: 12, + }, + dataType: { + type: "string", + id: 13, + }, + trans: { + rule: "repeated", + type: "Translation", + id: 14, + }, + culture: { + type: "string", + id: 15, + }, + textTs: { + type: "int64", + id: 16, + }, + }, + }, + Word: { + fields: { + text: { + type: "string", + id: 1, + }, + startMs: { + type: "int32", + id: 2, + }, + durationMs: { + type: "int32", + id: 3, + }, + isFinal: { + type: "bool", + id: 4, + }, + confidence: { + type: "double", + id: 5, + }, + }, + }, + Translation: { + fields: { + isFinal: { + type: "bool", + id: 1, + }, + lang: { + type: "string", + id: 2, + }, + texts: { + rule: "repeated", + type: "string", + id: 3, + }, + }, + }, + }, + }, + }, + }, + }, +) + +export { $root as default } diff --git a/demo/src/store/index.ts b/demo/src/store/index.ts new file mode 100644 index 00000000..8c6c1482 --- /dev/null +++ b/demo/src/store/index.ts @@ -0,0 +1,21 @@ +"use client" + +import globalReducer from "./reducers/global" +import { configureStore } from '@reduxjs/toolkit' + +export * from "./provider" + +export const makeStore = () => { + return configureStore({ + reducer: { + global: globalReducer, + }, + devTools: process.env.NODE_ENV !== "production", + }) +} + +// Infer the type of makeStore +export type AppStore = ReturnType +// Infer the `RootState` and `AppDispatch` types from the store itself +export type RootState = ReturnType +export type AppDispatch = AppStore['dispatch'] diff --git a/demo/src/store/provider/index.tsx b/demo/src/store/provider/index.tsx new file mode 100644 index 00000000..f34703b7 --- /dev/null +++ b/demo/src/store/provider/index.tsx @@ -0,0 +1,21 @@ +"use client"; + +import { useRef } from 'react' +import { Provider } from 'react-redux' +import { makeStore, AppStore } from '..' + +export function StoreProvider({ + children +}: { + children: React.ReactNode +}) { + const storeRef = useRef() + + if (!storeRef.current) { + // Create the store instance the first time this renders + storeRef.current = makeStore() + } + + return {children} +} + diff --git a/demo/src/store/reducers/global.ts b/demo/src/store/reducers/global.ts new file mode 100644 index 00000000..b29f0af8 --- /dev/null +++ b/demo/src/store/reducers/global.ts @@ -0,0 +1,105 @@ +import { IOptions, IChatItem, Language, VoiceType } from "@/types" +import { createSlice, PayloadAction } from "@reduxjs/toolkit" +import { DEFAULT_OPTIONS, COLOR_LIST, setOptionsToLocal, genRandomChatList } from "@/common" + +export interface InitialState { + options: IOptions + roomConnected: boolean, + agentConnected: boolean, + themeColor: string, + language: Language + voiceType: VoiceType + chatItems: IChatItem[], + graphName: string +} + +const getInitialState = (): InitialState => { + return { + options: DEFAULT_OPTIONS, + themeColor: COLOR_LIST[0].active, + roomConnected: false, + agentConnected: false, + language: "en-US", + voiceType: "male", + chatItems: [], + graphName: "camera.va.openai.azure" + } +} + +export const globalSlice = createSlice({ + name: "global", + initialState: getInitialState(), + reducers: { + setOptions: (state, action: PayloadAction>) => { + state.options = { ...state.options, ...action.payload } + setOptionsToLocal(state.options) + }, + setThemeColor: (state, action: PayloadAction) => { + state.themeColor = action.payload + document.documentElement.style.setProperty('--theme-color', action.payload); + }, + setRoomConnected: (state, action: PayloadAction) => { + state.roomConnected = action.payload + }, + addChatItem: (state, action: PayloadAction) => { + const { userId, text, isFinal, type, time } = action.payload + const LastFinalIndex = state.chatItems.findLastIndex((el) => { + return el.userId == userId && el.isFinal + }) + const LastNonFinalIndex = state.chatItems.findLastIndex((el) => { + return el.userId == userId && !el.isFinal + }) + let LastFinalItem = state.chatItems[LastFinalIndex] + let LastNonFinalItem = state.chatItems[LastNonFinalIndex] + if (LastFinalItem) { + // has last final Item + if (time <= LastFinalItem.time) { + // discard + console.log("[test] addChatItem, time < last final item, discard!:", text, isFinal, type) + return + } else { + if (LastNonFinalItem) { + console.log("[test] addChatItem, update last item(none final):", text, isFinal, type) + state.chatItems[LastNonFinalIndex] = action.payload + } else { + console.log("[test] addChatItem, add new item:", text, isFinal, type) + state.chatItems.push(action.payload) + } + } + } else { + // no last final Item + if (LastNonFinalItem) { + console.log("[test] addChatItem, update last item(none final):", text, isFinal, type) + state.chatItems[LastNonFinalIndex] = action.payload + } else { + console.log("[test] addChatItem, add new item:", text, isFinal, type) + state.chatItems.push(action.payload) + } + } + state.chatItems.sort((a, b) => a.time - b.time) + }, + setAgentConnected: (state, action: PayloadAction) => { + state.agentConnected = action.payload + }, + setLanguage: (state, action: PayloadAction) => { + state.language = action.payload + }, + setGraphName: (state, action: PayloadAction) => { + state.graphName = action.payload + }, + setVoiceType: (state, action: PayloadAction) => { + state.voiceType = action.payload + }, + reset: (state) => { + Object.assign(state, getInitialState()) + document.documentElement.style.setProperty('--theme-color', COLOR_LIST[0].active); + }, + }, +}) + +export const { reset, setOptions, + setRoomConnected, setAgentConnected, setVoiceType, + addChatItem, setThemeColor, setLanguage, setGraphName } = + globalSlice.actions + +export default globalSlice.reducer diff --git a/demo/src/types/index.ts b/demo/src/types/index.ts new file mode 100644 index 00000000..f5492003 --- /dev/null +++ b/demo/src/types/index.ts @@ -0,0 +1,62 @@ +export type Language = "en-US" | "zh-CN" | "ja-JP" | "ko-KR" +export type VoiceType = "male" | "female" + +export interface ColorItem { + active: string, + default: string +} + + +export interface IOptions { + channel: string, + userName: string, + userId: number +} + + +export interface IChatItem { + userId: number | string, + userName?: string, + text: string + type: "agent" | "user" + isFinal?: boolean + time: number +} + + +export interface ITextItem { + dataType: "transcribe" | "translate" + uid: string + time: number + text: string + isFinal: boolean +} + +export interface GraphOptionItem { + label: string + value: string +} + +export interface LanguageOptionItem { + label: string + value: Language +} + + +export interface VoiceOptionItem { + label: string + value: VoiceType +} + + +export interface OptionType { + value: string; + label: string; +} + + +export interface IPdfData { + fileName: string, + collection: string +} + diff --git a/demo/tsconfig.json b/demo/tsconfig.json new file mode 100644 index 00000000..15dcdd38 --- /dev/null +++ b/demo/tsconfig.json @@ -0,0 +1,40 @@ +{ + "compilerOptions": { + "lib": [ + "dom", + "dom.iterable", + "esnext" + ], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": [ + "./src/*" + ] + } + }, + "include": [ + "svgr.d.ts", + "next-env.d.ts", + "**/*.ts", + "**/*.tsx", + ".next/types/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/docker-compose.yml b/docker-compose.yml index c97e9e50..7b42c1b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: astra_agents_dev: - image: ghcr.io/ten-framework/astra_agents_build:0.4.0 + image: ghcr.io/ten-framework/ten_agent_build:0.1.0 container_name: astra_agents_dev platform: linux/amd64 tty: true @@ -19,7 +19,7 @@ services: networks: - astra_network astra_playground: - image: ghcr.io/ten-framework/astra_playground:v0.4.1-2-gb324bcd + image: ghcr.io/ten-framework/astra_playground:v0.4.1-9-g09a1df2 container_name: astra_playground restart: always ports: @@ -28,6 +28,17 @@ services: - astra_network environment: - AGENT_SERVER_URL=http://astra_agents_dev:8080 + - TEN_DEV_SERVER_URL=http://astra_agents_dev:49483 + agent_demo_ui: + image: ghcr.io/ten-framework/agent_demo:v0.4.1-9-g09a1df2 + container_name: agent_demo_ui + restart: always + ports: + - "3002:3000" + networks: + - astra_network + environment: + - AGENT_SERVER_URL=http://astra_agents_dev:8080 # use this when you want to run the playground in local development mode # astra_playground_dev: diff --git a/playground/.env b/playground/.env index 5f92b324..8739b9b0 100644 --- a/playground/.env +++ b/playground/.env @@ -1 +1,2 @@ -AGENT_SERVER_URL=http://localhost:8080 \ No newline at end of file +AGENT_SERVER_URL=http://localhost:8080 +TEN_DEV_SERVER_URL=http://localhost:49483 \ No newline at end of file diff --git a/playground/package.json b/playground/package.json index 664fb9f8..1a8d6f4f 100644 --- a/playground/package.json +++ b/playground/package.json @@ -10,32 +10,33 @@ "proto": "pbjs -t json-module -w commonjs -o src/protobuf/SttMessage.js src/protobuf/SttMessage.proto" }, "dependencies": { - "react": "^18", - "react-dom": "^18", + "@ant-design/icons": "^5.3.7", + "@reduxjs/toolkit": "^2.2.3", + "agora-rtc-sdk-ng": "^4.21.0", + "antd": "^5.15.3", + "axios": "^1.7.7", "next": "14.2.4", - "redux": "^5.0.1", "protobufjs": "^7.2.5", + "react": "^18", + "react-colorful": "^5.6.1", + "react-dom": "^18", "react-redux": "^9.1.0", - "@reduxjs/toolkit": "^2.2.3", - "antd": "^5.15.3", - "@ant-design/icons": "^5.3.7", - "agora-rtc-sdk-ng": "^4.21.0", - "react-colorful": "^5.6.1" + "redux": "^5.0.1" }, "devDependencies": { "@minko-fe/postcss-pxtoviewport": "^1.3.2", - "typescript": "^5", + "@svgr/webpack": "^8.1.0", "@types/node": "^20", "@types/react": "^18", "@types/react-dom": "^18", "@types/react-redux": "^7.1.22", + "autoprefixer": "^10.4.16", "eslint": "^8", "eslint-config-next": "14.2.4", - "autoprefixer": "^10.4.16", "postcss": "^8.4.31", + "protobufjs-cli": "^1.1.2", "sass": "^1.77.5", - "@svgr/webpack": "^8.1.0", - "protobufjs-cli": "^1.1.2" + "typescript": "^5" }, "packageManager": "yarn@1.22.22+sha1.ac34549e6aa8e7ead463a7407e1c7390f61a6610" -} \ No newline at end of file +} diff --git a/playground/src/app/api/agents/start/route.tsx b/playground/src/app/api/agents/start/route.tsx new file mode 100644 index 00000000..a346ea1f --- /dev/null +++ b/playground/src/app/api/agents/start/route.tsx @@ -0,0 +1,51 @@ +import { NextRequest, NextResponse } from 'next/server'; +import axios from 'axios'; + +/** + * Handles the POST request to start an agent. + * + * @param request - The NextRequest object representing the incoming request. + * @returns A NextResponse object representing the response to be sent back to the client. + */ +export async function POST(request: NextRequest) { + try { + const { AGENT_SERVER_URL } = process.env; + + // Check if environment variables are available + if (!AGENT_SERVER_URL) { + throw "Environment variables are not available"; + } + + const body = await request.json(); + const { + request_id, + channel_name, + user_uid, + graph_name, + language, + voice_type, + properties + } = body; + + // Send a POST request to start the agent + const response = await axios.post(`${AGENT_SERVER_URL}/start`, { + request_id, + channel_name, + user_uid, + graph_name, + // Get the graph properties based on the graph name, language, and voice type + properties: properties, + }); + + const responseData = response.data; + + return NextResponse.json(responseData, { status: response.status }); + } catch (error) { + if (error instanceof Response) { + const errorData = await error.json(); + return NextResponse.json(errorData, { status: error.status }); + } else { + return NextResponse.json({ code: "1", data: null, msg: "Internal Server Error" }, { status: 500 }); + } + } +} \ No newline at end of file diff --git a/playground/src/app/global.css b/playground/src/app/global.css index f7007287..7a1e1861 100644 --- a/playground/src/app/global.css +++ b/playground/src/app/global.css @@ -8,6 +8,7 @@ html, body { background-color: #0F0F11; font-family: "PingFang SC"; + height: 100%; } a { diff --git a/playground/src/app/page.tsx b/playground/src/app/page.tsx index 1bdcdeda..882aa4bd 100644 --- a/playground/src/app/page.tsx +++ b/playground/src/app/page.tsx @@ -1,14 +1,35 @@ -import LoginCard from "@/components/loginCard" -import styles from "./index.module.scss" +"use client" -export default function Login() { +import AuthInitializer from "@/components/authInitializer" +import { getRandomChannel, getRandomUserId, isMobile, useAppDispatch, useAppSelector } from "@/common" +import dynamic from 'next/dynamic' +import { useEffect, useState } from "react" +import { setOptions } from "@/store/reducers/global" + +const PCEntry = dynamic(() => import('@/platform/pc/entry'), { + ssr: false, +}) + +const MobileEntry = dynamic(() => import('@/platform/mobile/entry'), { + ssr: false, +}) + +export default function Home() { + const dispatch = useAppDispatch() + const [mobile, setMobile] = useState(null); + + + useEffect(() => { + setMobile(isMobile()) + }) return ( -
-
-
-
- -
+ mobile === null ? <> : + + {mobile ? : } + ); } + + + diff --git a/playground/src/common/hooks.ts b/playground/src/common/hooks.ts index 9759fa29..ff1d8050 100644 --- a/playground/src/common/hooks.ts +++ b/playground/src/common/hooks.ts @@ -129,3 +129,16 @@ export const usePrevious = (value: any) => { }; +export const useGraphExtensions = () => { + const graphName = useAppSelector(state => state.global.graphName); + const nodes = useAppSelector(state => state.global.extensions); + const [graphExtensions, setGraphExtensions] = useState>({}); + + useEffect(() => { + if (nodes && nodes[graphName]) { + setGraphExtensions(nodes[graphName]); + } + }, [graphName, nodes]); + + return graphExtensions; +}; \ No newline at end of file diff --git a/playground/src/common/request.ts b/playground/src/common/request.ts index a6acda1f..ca8b566f 100644 --- a/playground/src/common/request.ts +++ b/playground/src/common/request.ts @@ -1,5 +1,6 @@ import { genUUID } from "./utils" import { Language } from "@/types" +import axios from "axios" interface StartRequestConfig { channel: string @@ -7,6 +8,7 @@ interface StartRequestConfig { graphName: string, language: Language, voiceType: "male" | "female" + properties: Record } interface GenAgoraDataConfig { @@ -23,37 +25,26 @@ export const apiGenAgoraData = async (config: GenAgoraDataConfig) => { uid: userId, channel_name: channel } - let resp: any = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(data), - }) - resp = (await resp.json()) || {} + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} return resp } export const apiStartService = async (config: StartRequestConfig): Promise => { // look at app/apis/route.tsx for the server-side implementation const url = `/api/agents/start` - const { channel, userId, graphName, language, voiceType } = config + const { channel, userId, graphName, language, voiceType, properties } = config const data = { request_id: genUUID(), channel_name: channel, user_uid: userId, graph_name: graphName, language, - voice_type: voiceType + voice_type: voiceType, + properties } - let resp: any = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(data), - }) - resp = (await resp.json()) || {} + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} return resp } @@ -64,27 +55,16 @@ export const apiStopService = async (channel: string) => { request_id: genUUID(), channel_name: channel } - let resp = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(data), - }) - resp = (await resp.json()) || {} + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} return resp } export const apiGetDocumentList = async () => { // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL const url = `/api/vector/document/preset/list` - let resp: any = await fetch(url, { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - }) - resp = (await resp.json()) || {} + let resp: any = await axios.get(url) + resp = (resp.data) || {} if (resp.code !== "0") { throw new Error(resp.msg) } @@ -101,14 +81,8 @@ export const apiUpdateDocument = async (options: { channel: string, collection: collection: collection, file_name: fileName } - let resp: any = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(data), - }) - resp = (await resp.json()) || {} + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} return resp } @@ -121,13 +95,31 @@ export const apiPing = async (channel: string) => { request_id: genUUID(), channel_name: channel } - let resp = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(data), - }) - resp = (await resp.json()) || {} + let resp: any = await axios.post(url, data) + resp = (resp.data) || {} return resp } + +export const apiGetGraphs = async () => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/dev/v1/graphs` + let resp: any = await axios.get(url) + resp = (resp.data) || {} + return resp +} + +export const apiGetExtensionMetadata = async () => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/dev/v1/addons/extensions` + let resp: any = await axios.get(url) + resp = (resp.data) || {} + return resp +} + +export const apiGetNodes = async (graphName: string) => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/dev/v1/graphs/${graphName}/nodes` + let resp: any = await axios.get(url) + resp = (resp.data) || {} + return resp +} \ No newline at end of file diff --git a/playground/src/components/authInitializer/index.tsx b/playground/src/components/authInitializer/index.tsx index 5ef763a1..65336e21 100644 --- a/playground/src/components/authInitializer/index.tsx +++ b/playground/src/components/authInitializer/index.tsx @@ -1,7 +1,7 @@ "use client" import { ReactNode, useEffect } from "react" -import { useAppDispatch, getOptionsFromLocal } from "@/common" +import { useAppDispatch, getOptionsFromLocal, getRandomChannel, getRandomUserId } from "@/common" import { setOptions, reset } from "@/store/reducers/global" interface AuthInitializerProps { @@ -15,9 +15,15 @@ const AuthInitializer = (props: AuthInitializerProps) => { useEffect(() => { if (typeof window !== "undefined") { const options = getOptionsFromLocal() - if (options) { + if (options && options.channel) { dispatch(reset()) dispatch(setOptions(options)) + } else { + dispatch(reset()) + dispatch(setOptions({ + channel: getRandomChannel(), + userId: getRandomUserId(), + })) } } }, [dispatch]) diff --git a/playground/src/components/pdfSelect/upload/index.tsx b/playground/src/components/pdfSelect/upload/index.tsx index 734b4a07..4eab9684 100644 --- a/playground/src/components/pdfSelect/upload/index.tsx +++ b/playground/src/components/pdfSelect/upload/index.tsx @@ -22,6 +22,7 @@ const PdfUpload = (props: PdfSelectProps) => { accept: "application/pdf", maxCount: 1, showUploadList: false, + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL action: `/api/vector/document/upload`, data: { channel_name: channel, diff --git a/playground/src/manager/rtc/rtc.ts b/playground/src/manager/rtc/rtc.ts index 0439be78..4139d89e 100644 --- a/playground/src/manager/rtc/rtc.ts +++ b/playground/src/manager/rtc/rtc.ts @@ -103,57 +103,82 @@ export class RtcManager extends AGEventEmitter { }) }) this.client.on("stream-message", (uid: UID, stream: any) => { - this._praseData(stream) + this._parseData(stream) }) } - private _praseData(data: any): ITextItem | void { - // @ts-ignore - // const textstream = protoRoot.Agora.SpeechToText.lookup("Text").decode(data) - // if (!textstream) { - // return console.warn("Prase data failed.") - // } - let decoder = new TextDecoder('utf-8') - let decodedMessage = decoder.decode(data) - - const textstream = JSON.parse(decodedMessage) - - console.log("[test] textstream raw data", JSON.stringify(textstream)) - const { stream_id, is_final, text, text_ts, data_type } = textstream - let textStr: string = "" - let isFinal = false - const textItem: ITextItem = {} as ITextItem - textItem.uid = stream_id - textItem.time = text_ts - // switch (dataType) { - // case "transcribe": - // words.forEach((word: any) => { - // textStr += word.text - // if (word.isFinal) { - // isFinal = true - // } - // }) - textItem.dataType = "transcribe" - // textItem.language = culture - textItem.text = text - textItem.isFinal = is_final - this.emit("textChanged", textItem) - // break - // case "translate": - // if (!trans?.length) { - // return - // } - // trans.forEach((transItem: any) => { - // textStr = transItem.texts.join("") - // isFinal = !!transItem.isFinal - // textItem.dataType = "translate" - // textItem.language = transItem.lang - // textItem.isFinal = isFinal - // textItem.text = textStr - // this.emit("textChanged", textItem) - // }) - // break - // } + private _parseData(data: any): ITextItem | void { + let decoder = new TextDecoder('utf-8'); + let decodedMessage = decoder.decode(data); + const textstream = JSON.parse(decodedMessage); + + console.log("[test] textstream raw data", JSON.stringify(textstream)); + + const { stream_id, is_final, text, text_ts, data_type, message_id, part_number, total_parts } = textstream; + + if (total_parts > 0) { + // If message is split, handle it accordingly + this._handleSplitMessage(message_id, part_number, total_parts, stream_id, is_final, text, text_ts); + } else { + // If there is no message_id, treat it as a complete message + this._handleCompleteMessage(stream_id, is_final, text, text_ts); + } + } + + private messageCache: { [key: string]: { parts: string[], totalParts: number } } = {}; + + /** + * Handle complete messages (not split). + */ + private _handleCompleteMessage(stream_id: number, is_final: boolean, text: string, text_ts: number): void { + const textItem: ITextItem = { + uid: `${stream_id}`, + time: text_ts, + dataType: "transcribe", + text: text, + isFinal: is_final + }; + + if (text.trim().length > 0) { + this.emit("textChanged", textItem); + } + } + + /** + * Handle split messages, track parts, and reassemble once all parts are received. + */ + private _handleSplitMessage( + message_id: string, + part_number: number, + total_parts: number, + stream_id: number, + is_final: boolean, + text: string, + text_ts: number + ): void { + // Ensure the messageCache entry exists for this message_id + if (!this.messageCache[message_id]) { + this.messageCache[message_id] = { parts: [], totalParts: total_parts }; + } + + const cache = this.messageCache[message_id]; + + // Store the received part at the correct index (part_number starts from 1, so we use part_number - 1) + cache.parts[part_number - 1] = text; + + // Check if all parts have been received + const receivedPartsCount = cache.parts.filter(part => part !== undefined).length; + + if (receivedPartsCount === total_parts) { + // All parts have been received, reassemble the message + const fullText = cache.parts.join(''); + + // Now that the message is reassembled, handle it like a complete message + this._handleCompleteMessage(stream_id, is_final, fullText, text_ts); + + // Remove the cached message since it is now fully processed + delete this.messageCache[message_id]; + } } diff --git a/playground/src/middleware.tsx b/playground/src/middleware.tsx index 66e3a549..0c40f24f 100644 --- a/playground/src/middleware.tsx +++ b/playground/src/middleware.tsx @@ -3,16 +3,20 @@ import { NextRequest, NextResponse } from 'next/server'; import { startAgent } from './apis/routes'; -const { AGENT_SERVER_URL } = process.env; +const { AGENT_SERVER_URL, TEN_DEV_SERVER_URL } = process.env; // Check if environment variables are available if (!AGENT_SERVER_URL) { - throw "Environment variables AGENT_SERVER_URL are not available"; + throw "Environment variables AGENT_SERVER_URL are not available"; +} + +if (!TEN_DEV_SERVER_URL) { + throw "Environment variables TEN_DEV_SERVER_URL are not available"; } export function middleware(req: NextRequest) { - const { pathname } = req.nextUrl; - const url = req.nextUrl.clone(); + const { pathname } = req.nextUrl; + const url = req.nextUrl.clone(); if (pathname.startsWith(`/api/agents/`)) { if (!pathname.startsWith('/api/agents/start')) { @@ -22,7 +26,7 @@ export function middleware(req: NextRequest) { // console.log(`Rewriting request to ${url.href}`); return NextResponse.rewrite(url); } else { - return startAgent(req); + return NextResponse.next(); } } else if (pathname.startsWith(`/api/vector/`)) { @@ -35,6 +39,14 @@ export function middleware(req: NextRequest) { // Proxy all other documents requests url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/token/', '/token/')}`; + // console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + } else if (pathname.startsWith('/api/dev/')) { + + // Proxy all other documents requests + const url = req.nextUrl.clone(); + url.href = `${TEN_DEV_SERVER_URL}${pathname.replace('/api/dev/', '/api/dev-server/')}`; + // console.log(`Rewriting request to ${url.href}`); return NextResponse.rewrite(url); } else { diff --git a/playground/src/platform/mobile/description/index.tsx b/playground/src/platform/mobile/description/index.tsx index 7473d550..bae88001 100644 --- a/playground/src/platform/mobile/description/index.tsx +++ b/playground/src/platform/mobile/description/index.tsx @@ -1,6 +1,6 @@ import { setAgentConnected } from "@/store/reducers/global" import { - DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, + useAppDispatch, useAppSelector, apiPing, genUUID, apiStartService, apiStopService } from "@/common" import { message } from "antd" @@ -50,7 +50,8 @@ const Description = () => { userId, graphName, language, - voiceType + voiceType, + properties: {} }) const { code, msg } = res || {} if (code != 0) { diff --git a/playground/src/platform/mobile/rtc/agent/index.module.scss b/playground/src/platform/mobile/rtc/agent/index.module.scss index fa3ae2ec..5bebb2bc 100644 --- a/playground/src/platform/mobile/rtc/agent/index.module.scss +++ b/playground/src/platform/mobile/rtc/agent/index.module.scss @@ -1,7 +1,6 @@ .agent { position: relative; display: flex; - height: 292px; padding: 20px 16px; flex-direction: column; justify-content: flex-start; @@ -22,7 +21,6 @@ } .view { - margin-top: 32px; display: flex; align-items: center; justify-content: center; diff --git a/playground/src/platform/mobile/rtc/agent/index.tsx b/playground/src/platform/mobile/rtc/agent/index.tsx index a7fd7944..159f6730 100644 --- a/playground/src/platform/mobile/rtc/agent/index.tsx +++ b/playground/src/platform/mobile/rtc/agent/index.tsx @@ -15,7 +15,6 @@ const Agent = (props: AgentProps) => { const subscribedVolumes = useMultibandTrackVolume(audioTrack, 12); return
-
Agent
{ {/* you */}
-
You
{/* microphone */} {/* camera */} diff --git a/playground/src/platform/pc/chat/index.tsx b/playground/src/platform/pc/chat/index.tsx index 64dea171..3ee02155 100644 --- a/playground/src/platform/pc/chat/index.tsx +++ b/playground/src/platform/pc/chat/index.tsx @@ -7,50 +7,78 @@ import { LANGUAGE_OPTIONS, useAppSelector, GRAPH_OPTIONS, isRagGraph, + apiGetGraphs, + apiGetNodes, + useGraphExtensions, + apiGetExtensionMetadata, } from "@/common" -import { setGraphName, setLanguage } from "@/store/reducers/global" -import { Select, } from 'antd'; +import { setExtensionMetadata, setGraphName, setGraphs, setLanguage, setExtensions } from "@/store/reducers/global" +import { Button, Modal, Select, Tabs, TabsProps, } from 'antd'; import PdfSelect from "@/components/pdfSelect" import styles from "./index.module.scss" - - +import { SettingOutlined } from "@ant-design/icons" +import EditableTable from "./table" const Chat = () => { const dispatch = useAppDispatch() - const chatItems = useAppSelector(state => state.global.chatItems) - const language = useAppSelector(state => state.global.language) + const graphs = useAppSelector(state => state.global.graphs) + const extensions = useAppSelector(state => state.global.extensions) const graphName = useAppSelector(state => state.global.graphName) + const chatItems = useAppSelector(state => state.global.chatItems) const agentConnected = useAppSelector(state => state.global.agentConnected) + const [modal2Open, setModal2Open] = useState(false) + const graphExtensions = useGraphExtensions() + const extensionMetadata = useAppSelector(state => state.global.extensionMetadata) + // const chatItems = genRandomChatList(10) const chatRef = useRef(null) + useEffect(() => { + apiGetGraphs().then((res: any) => { + let graphs = res["data"].map((item: any) => item["name"]) + dispatch(setGraphs(graphs)) + }) + apiGetExtensionMetadata().then((res: any) => { + let metadata = res["data"] + let metadataMap: Record = {} + metadata.forEach((item: any) => { + metadataMap[item["name"]] = item + }) + dispatch(setExtensionMetadata(metadataMap)) + }) + }, []) + + useEffect(() => { + if (!extensions[graphName]) { + apiGetNodes(graphName).then((res: any) => { + let nodes = res["data"] + let nodesMap: Record = {} + nodes.forEach((item: any) => { + nodesMap[item["name"]] = item + }) + dispatch(setExtensions({ graphName, nodesMap })) + }) + } + }, [graphName]) useAutoScroll(chatRef) - - const onLanguageChange = (val: any) => { - dispatch(setLanguage(val)) - } - const onGraphNameChange = (val: any) => { dispatch(setGraphName(val)) } - return
- + {isRagGraph(graphName) ? : null}
@@ -59,6 +87,36 @@ const Chat = () => { return })}
+ setModal2Open(false)} + footer={ + + } + > +

You can adjust extension properties here, the values will be overridden when the agent starts using "Connect." Note that this won't modify the property.json file.

+ { + let node = graphExtensions[key] + let addon = node["addon"] + let metadata = extensionMetadata[addon] + return { + key: node["name"], label: node["name"], children: { + let nodesMap = JSON.parse(JSON.stringify(graphExtensions)) + nodesMap[key]["property"] = data + dispatch(setExtensions({ graphName, nodesMap })) + }} + > + } + })} /> + } diff --git a/playground/src/platform/pc/chat/table/index.tsx b/playground/src/platform/pc/chat/table/index.tsx new file mode 100644 index 00000000..944dedd4 --- /dev/null +++ b/playground/src/platform/pc/chat/table/index.tsx @@ -0,0 +1,168 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { Button, Empty, ConfigProvider, Table, Input, Form, Checkbox, InputRef } from 'antd'; +import type { ColumnsType } from 'antd/es/table'; + +// Define the data type for the table rows +interface DataType { + key: string; + value: string | number | boolean | null; +} + +// Define the props for the EditableTable component +interface EditableTableProps { + initialData: Record; + onUpdate: (updatedData: Record) => void; + metadata: Record; // Metadata with property types +} + +// Helper to convert values based on type +const convertToType = (value: any, type: string) => { + switch (type) { + case 'int64': + case 'int32': + return parseInt(value, 10); + case 'float64': + return parseFloat(value); + case 'bool': + return value === true || value === 'true'; + case 'string': + return String(value); + default: + return value; + } +}; + +const EditableTable: React.FC = ({ initialData, onUpdate, metadata }) => { + const [dataSource, setDataSource] = useState( + Object.entries(initialData).map(([key, value]) => ({ key, value })) + ); + const [editingKey, setEditingKey] = useState(''); + const [form] = Form.useForm(); + const inputRef = useRef(null); // Ref to manage focus + + // Function to check if the current row is being edited + const isEditing = (record: DataType) => record.key === editingKey; + + // Function to toggle editing on a row + const edit = (record: DataType) => { + form.setFieldsValue({ value: record.value ?? '' }); + setEditingKey(record.key); + }; + + // Function to handle when the value of a non-boolean field is changed + const handleValueChange = async (key: string) => { + try { + const row = await form.validateFields(); + const newData = [...dataSource]; + const index = newData.findIndex((item) => key === item.key); + + if (index > -1) { + const item = newData[index]; + const valueType = metadata[key]?.type || 'string'; + newData.splice(index, 1, { ...item, ...row, value: convertToType(row.value, valueType) }); + setDataSource(newData); + setEditingKey(''); + + // Notify the parent component of the update + const updatedData = Object.fromEntries(newData.map(({ key, value }) => [key, value])); + onUpdate(updatedData); + } + } catch (errInfo) { + console.log('Validation Failed:', errInfo); + } + }; + + // Toggle the checkbox for boolean values directly in the table cell + const handleCheckboxChange = (key: string, checked: boolean) => { + const newData = [...dataSource]; + const index = newData.findIndex((item) => key === item.key); + if (index > -1) { + newData[index].value = checked; // Update the boolean value + setDataSource(newData); + + // Notify the parent component of the update + const updatedData = Object.fromEntries(newData.map(({ key, value }) => [key, value])); + onUpdate(updatedData); + } + }; + + // Auto-focus on the input when entering edit mode + useEffect(() => { + if (editingKey) { + inputRef.current?.focus(); // Focus the input field when editing starts + } + }, [editingKey]); + + // Define columns for the table + const columns: ColumnsType = [ + { + title: 'Key', + dataIndex: 'key', + width: '30%', + key: 'key', + }, + { + title: 'Value', + dataIndex: 'value', + width: '50%', + key: 'value', + render: (_, record: DataType) => { + const valueType = metadata[record.key]?.type || 'string'; + + // Always display the checkbox for boolean values + if (valueType === 'bool') { + return ( + handleCheckboxChange(record.key, e.target.checked)} + /> + ); + } + + // Inline editing for other types (string, number) + const editable = isEditing(record); + return editable ? ( + + handleValueChange(record.key)} // Save on pressing Enter + onBlur={() => handleValueChange(record.key)} // Save on losing focus + /> + + ) : ( +
edit(record)}> + {record.value !== null && record.value !== undefined + ? record.value + : 'Click to edit'} +
// Display placeholder for empty values + ); + }, + }, + ]; + + return ( + ( + + + )} + > +
+ + + + ); +}; + +export default EditableTable; diff --git a/playground/src/platform/pc/description/index.tsx b/playground/src/platform/pc/description/index.tsx index 01f61063..53d7d996 100644 --- a/playground/src/platform/pc/description/index.tsx +++ b/playground/src/platform/pc/description/index.tsx @@ -1,7 +1,8 @@ import { setAgentConnected } from "@/store/reducers/global" import { - DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, - apiStartService, apiStopService + useAppDispatch, useAppSelector, apiPing, genUUID, + apiStartService, apiStopService, + useGraphExtensions } from "@/common" import { Select, Button, message, Upload } from "antd" import { useEffect, useState, MouseEventHandler } from "react" @@ -17,8 +18,9 @@ const Description = () => { const userId = useAppSelector(state => state.global.options.userId) const language = useAppSelector(state => state.global.language) const voiceType = useAppSelector(state => state.global.voiceType) - const graphName = useAppSelector(state => state.global.graphName) const [loading, setLoading] = useState(false) + const graphName = useAppSelector(state => state.global.graphName) + const graphNodes = useGraphExtensions() useEffect(() => { if (channel) { @@ -45,12 +47,18 @@ const Description = () => { message.success("Agent disconnected") stopPing() } else { + let properties: Record = {} + Object.keys(graphNodes).forEach(extensionName => { + properties[extensionName] = {} + properties[extensionName] = graphNodes[extensionName].property + }) const res = await apiStartService({ channel, userId, graphName, language, - voiceType + voiceType, + properties: properties }) const { code, msg } = res || {} if (code != 0) { diff --git a/playground/src/platform/pc/entry/index.module.scss b/playground/src/platform/pc/entry/index.module.scss index f138183f..bb001f27 100644 --- a/playground/src/platform/pc/entry/index.module.scss +++ b/playground/src/platform/pc/entry/index.module.scss @@ -2,16 +2,27 @@ position: relative; height: 100%; box-sizing: border-box; + display: flex; + flex-direction: column; .content { position: relative; padding: 16px; box-sizing: border-box; + display: flex; + height: calc(100% - 64px); + flex-direction: column; .body { margin-top: 16px; display: flex; gap: 24px; + flex-grow: 1; + + .chat { + display: flex; + flex-grow: 1; + } } } -} +} \ No newline at end of file diff --git a/playground/src/platform/pc/entry/index.tsx b/playground/src/platform/pc/entry/index.tsx index e7acd7f1..a7ee7592 100644 --- a/playground/src/platform/pc/entry/index.tsx +++ b/playground/src/platform/pc/entry/index.tsx @@ -11,8 +11,12 @@ const PCEntry = () => {
- - +
+ +
+
+ +
diff --git a/playground/src/platform/pc/rtc/agent/index.module.scss b/playground/src/platform/pc/rtc/agent/index.module.scss index fa3ae2ec..5bebb2bc 100644 --- a/playground/src/platform/pc/rtc/agent/index.module.scss +++ b/playground/src/platform/pc/rtc/agent/index.module.scss @@ -1,7 +1,6 @@ .agent { position: relative; display: flex; - height: 292px; padding: 20px 16px; flex-direction: column; justify-content: flex-start; @@ -22,7 +21,6 @@ } .view { - margin-top: 32px; display: flex; align-items: center; justify-content: center; diff --git a/playground/src/platform/pc/rtc/agent/index.tsx b/playground/src/platform/pc/rtc/agent/index.tsx index a7fd7944..159f6730 100644 --- a/playground/src/platform/pc/rtc/agent/index.tsx +++ b/playground/src/platform/pc/rtc/agent/index.tsx @@ -15,7 +15,6 @@ const Agent = (props: AgentProps) => { const subscribedVolumes = useMultibandTrackVolume(audioTrack, 12); return
-
Agent
{ const dispatch = useAppDispatch() const options = useAppSelector(state => state.global.options) - const voiceType = useAppSelector(state => state.global.voiceType) - const agentConnected = useAppSelector(state => state.global.agentConnected) const { userId, channel } = options const [videoTrack, setVideoTrack] = useState() const [audioTrack, setAudioTrack] = useState() @@ -97,25 +93,15 @@ const Rtc = () => { } } - const onVoiceChange = (value: any) => { - dispatch(setVoiceType(value)) - } - return
Audio & Video - } - options={VOICE_OPTIONS} onChange={onVoiceChange}>
{/* agent */} {/* you */}
-
You
{/* microphone */} {/* camera */} diff --git a/playground/src/store/reducers/global.ts b/playground/src/store/reducers/global.ts index b29f0af8..dbdbff3b 100644 --- a/playground/src/store/reducers/global.ts +++ b/playground/src/store/reducers/global.ts @@ -10,7 +10,10 @@ export interface InitialState { language: Language voiceType: VoiceType chatItems: IChatItem[], - graphName: string + graphName: string, + graphs: string[], + extensions: Record, + extensionMetadata: Record } const getInitialState = (): InitialState => { @@ -22,7 +25,10 @@ const getInitialState = (): InitialState => { language: "en-US", voiceType: "male", chatItems: [], - graphName: "camera.va.openai.azure" + graphName: "camera.va.openai.azure", + graphs: [], + extensions: {}, + extensionMetadata: {}, } } @@ -87,6 +93,16 @@ export const globalSlice = createSlice({ setGraphName: (state, action: PayloadAction) => { state.graphName = action.payload }, + setGraphs: (state, action: PayloadAction) => { + state.graphs = action.payload + }, + setExtensions: (state, action: PayloadAction>) => { + let { graphName, nodesMap } = action.payload + state.extensions[graphName] = nodesMap + }, + setExtensionMetadata: (state, action: PayloadAction>) => { + state.extensionMetadata = action.payload + }, setVoiceType: (state, action: PayloadAction) => { state.voiceType = action.payload }, @@ -99,7 +115,7 @@ export const globalSlice = createSlice({ export const { reset, setOptions, setRoomConnected, setAgentConnected, setVoiceType, - addChatItem, setThemeColor, setLanguage, setGraphName } = + addChatItem, setThemeColor, setLanguage, setGraphName, setGraphs, setExtensions, setExtensionMetadata } = globalSlice.actions export default globalSlice.reducer diff --git a/playground/src/types/index.ts b/playground/src/types/index.ts index 02c6bd70..f5492003 100644 --- a/playground/src/types/index.ts +++ b/playground/src/types/index.ts @@ -27,7 +27,6 @@ export interface IChatItem { export interface ITextItem { dataType: "transcribe" | "translate" uid: string - language: string time: number text: string isFinal: boolean diff --git a/server/internal/http_server.go b/server/internal/http_server.go index f3fea8b2..d7c41915 100644 --- a/server/internal/http_server.go +++ b/server/internal/http_server.go @@ -15,7 +15,6 @@ import ( "net/http" "os" "path/filepath" - "regexp" "strings" "time" @@ -381,23 +380,6 @@ func (s *HttpServer) output(c *gin.Context, code *Code, data any, httpStatus ... c.JSON(httpStatus[0], gin.H{"code": code.code, "msg": code.msg, "data": data}) } -func replaceEnvVarsInJSON(jsonData string) string { - // Regex to find all occurrences of $VAR_NAME - re := regexp.MustCompile(`"\$(\w+)"`) - - // Function to replace the match with the environment variable value - result := re.ReplaceAllStringFunc(jsonData, func(match string) string { - // Extract the variable name (removing the leading $ and surrounding quotes) - envVar := strings.Trim(match, "\"$") - // Get the environment variable value - value := os.Getenv(envVar) - // Replace with the value (keeping it quoted) - return fmt.Sprintf("\"%s\"", value) - }) - - return result -} - func (s *HttpServer) processProperty(req *StartReq) (propertyJsonFile string, logFile string, err error) { content, err := os.ReadFile(PropertyJsonFile) if err != nil { @@ -430,10 +412,10 @@ func (s *HttpServer) processProperty(req *StartReq) (propertyJsonFile string, lo graphs := gjson.Get(propertyJson, "_ten.predefined_graphs").Array() // Create a new array for graphs that match the name - var newGraphs []string + var newGraphs []gjson.Result for _, graph := range graphs { if graph.Get("name").String() == graphName { - newGraphs = append(newGraphs, graph.Raw) + newGraphs = append(newGraphs, graph) } } @@ -443,20 +425,28 @@ func (s *HttpServer) processProperty(req *StartReq) (propertyJsonFile string, lo return } + // Set the array of graphs directly using sjson.Set + graphData := make([]interface{}, len(newGraphs)) + for i, graph := range newGraphs { + graphData[i] = graph.Value() // Convert gjson.Result to interface{} + } + // Replace the predefined_graphs array with the filtered array - propertyJson, _ = sjson.SetRaw(propertyJson, "_ten.predefined_graphs", fmt.Sprintf("[%s]", strings.Join(newGraphs, ","))) + propertyJson, _ = sjson.Set(propertyJson, "_ten.predefined_graphs", graphData) // Automatically start on launch propertyJson, _ = sjson.Set(propertyJson, fmt.Sprintf(`%s.auto_start`, graph), true) - // Set environment variable values to property.json - propertyJson = replaceEnvVarsInJSON(propertyJson) - // Set additional properties to property.json for extensionName, props := range req.Properties { if extKey := extensionName; extKey != "" { for prop, val := range props { - propertyJson, _ = sjson.Set(propertyJson, fmt.Sprintf(`%s.nodes.#(name=="%s").property.%s`, graph, extKey, prop), val) + // Construct the path + path := fmt.Sprintf(`%s.nodes.#(name=="%s").property.%s`, graph, extKey, prop) + propertyJson, err = sjson.Set(propertyJson, path, val) + if err != nil { + slog.Error("handlerStart set property failed", "err", err, "graph", graphName, "extensionName", extensionName, "prop", prop, "val", val, "requestId", req.RequestId, logTag) + } } } }