diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index df45ecc6..8f401d9f 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -1,53 +1,63 @@ name: Build Docker -on: +on: push: - branches: [ "main" ] + branches: ["main"] # Publish semver tags as releases. - tags: [ 'v*.*.*' ] + tags: ["v*.*.*"] paths-ignore: - - '.devcontainer/**' - - '.github/ISSUE_TEMPLATE/**' - - 'images/**' - - 'playground/**' - - '**.md' + - ".devcontainer/**" + - ".github/ISSUE_TEMPLATE/**" + - "images/**" + - "playground/**" + - "**.md" pull_request: - branches: [ "main" ] + branches: ["main"] workflow_dispatch: env: SERVER_IMAGE_NAME: astra_agents_server PLAYGROUND_IMAGE_NAME: astra_playground + DEMO_IMAGE_NAME: agent_demo jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-tags: true - fetch-depth: '0' - - id: pre-step - shell: bash - run: echo "image-tag=$(git describe --tags --always)" >> $GITHUB_OUTPUT - - name: Build & Publish Docker Image for Agents Server - uses: elgohr/Publish-Docker-Github-Action@v5 - with: - name: ${{ github.repository_owner }}/${{ env.SERVER_IMAGE_NAME }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ghcr.io - tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" - no_push: ${{ github.event_name == 'pull_request' }} - - name: Build & Publish Docker Image for Playground - uses: elgohr/Publish-Docker-Github-Action@v5 - with: - name: ${{ github.repository_owner }}/${{ env.PLAYGROUND_IMAGE_NAME }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ghcr.io - workdir: playground - tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" - no_push: ${{ github.event_name == 'pull_request' }} - + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-tags: true + fetch-depth: "0" + - id: pre-step + shell: bash + run: echo "image-tag=$(git describe --tags --always)" >> $GITHUB_OUTPUT + - name: Build & Publish Docker Image for Agents Server + uses: elgohr/Publish-Docker-Github-Action@v5 + with: + name: ${{ github.repository_owner }}/${{ env.SERVER_IMAGE_NAME }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" + no_push: ${{ github.event_name == 'pull_request' }} + - name: Build & Publish Docker Image for Playground + uses: elgohr/Publish-Docker-Github-Action@v5 + with: + name: ${{ github.repository_owner }}/${{ env.PLAYGROUND_IMAGE_NAME }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + workdir: playground + tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" + no_push: ${{ github.event_name == 'pull_request' }} + - name: Build & Publish Docker Image for demo + uses: elgohr/Publish-Docker-Github-Action@v5 + with: + name: ${{ github.repository_owner }}/${{ env.DEMO_IMAGE_NAME }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + workdir: demo + tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}" + no_push: ${{ github.event_name == 'pull_request' }} diff --git a/demo/.dockerignore b/demo/.dockerignore new file mode 100644 index 00000000..80ae13ce --- /dev/null +++ b/demo/.dockerignore @@ -0,0 +1,3 @@ +.git +.next +node_modules diff --git a/demo/.env b/demo/.env new file mode 100644 index 00000000..5f92b324 --- /dev/null +++ b/demo/.env @@ -0,0 +1 @@ +AGENT_SERVER_URL=http://localhost:8080 \ No newline at end of file diff --git a/demo/.gitignore b/demo/.gitignore new file mode 100644 index 00000000..3bcf2bf5 --- /dev/null +++ b/demo/.gitignore @@ -0,0 +1,135 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +# .env +!.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# lock +package-lock.json +yarn.lock diff --git a/demo/Dockerfile b/demo/Dockerfile new file mode 100644 index 00000000..63379d47 --- /dev/null +++ b/demo/Dockerfile @@ -0,0 +1,27 @@ +FROM node:20-alpine AS base + +FROM base AS builder + +WORKDIR /app + +# COPY .env.example .env +COPY . . + +RUN npm i --verbose && \ + npm run build + + +FROM base AS runner + +WORKDIR /app + +ENV NODE_ENV production + +RUN mkdir .next + +COPY --from=builder /app/.next/standalone ./ +COPY --from=builder /app/.next/static ./.next/static + +EXPOSE 3000 + +CMD HOSTNAME="0.0.0.0" node server.js \ No newline at end of file diff --git a/demo/LICENSE b/demo/LICENSE new file mode 100644 index 00000000..e4589a2b --- /dev/null +++ b/demo/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Agora Community + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/demo/next-env.d.ts b/demo/next-env.d.ts new file mode 100644 index 00000000..4f11a03d --- /dev/null +++ b/demo/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/basic-features/typescript for more information. diff --git a/demo/next.config.mjs b/demo/next.config.mjs new file mode 100644 index 00000000..132f5fc1 --- /dev/null +++ b/demo/next.config.mjs @@ -0,0 +1,37 @@ +/** @type {import('next').NextConfig} */ + +const nextConfig = { + // basePath: '/ai-agent', + // output: 'export', + output: 'standalone', + reactStrictMode: false, + webpack(config) { + // Grab the existing rule that handles SVG imports + const fileLoaderRule = config.module.rules.find((rule) => + rule.test?.test?.('.svg'), + ) + + config.module.rules.push( + // Reapply the existing rule, but only for svg imports ending in ?url + { + ...fileLoaderRule, + test: /\.svg$/i, + resourceQuery: /url/, // *.svg?url + }, + // Convert all other *.svg imports to React components + { + test: /\.svg$/i, + issuer: fileLoaderRule.issuer, + resourceQuery: { not: [...fileLoaderRule.resourceQuery.not, /url/] }, // exclude if *.svg?url + use: ['@svgr/webpack'], + }, + ) + + // Modify the file loader rule to ignore *.svg, since we have it handled now. + fileLoaderRule.exclude = /\.svg$/i + + return config + } +}; + +export default nextConfig; diff --git a/demo/package.json b/demo/package.json new file mode 100644 index 00000000..4342a11d --- /dev/null +++ b/demo/package.json @@ -0,0 +1,41 @@ +{ + "name": "astra.ai-playground", + "version": "0.4.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint", + "proto": "pbjs -t json-module -w commonjs -o src/protobuf/SttMessage.js src/protobuf/SttMessage.proto" + }, + "dependencies": { + "react": "^18", + "react-dom": "^18", + "next": "14.2.4", + "redux": "^5.0.1", + "protobufjs": "^7.2.5", + "react-redux": "^9.1.0", + "@reduxjs/toolkit": "^2.2.3", + "antd": "^5.15.3", + "@ant-design/icons": "^5.3.7", + "agora-rtc-sdk-ng": "^4.21.0", + "react-colorful": "^5.6.1" + }, + "devDependencies": { + "@minko-fe/postcss-pxtoviewport": "^1.3.2", + "typescript": "^5", + "@types/node": "^20", + "@types/react": "^18", + "@types/react-dom": "^18", + "@types/react-redux": "^7.1.22", + "eslint": "^8", + "eslint-config-next": "14.2.4", + "autoprefixer": "^10.4.16", + "postcss": "^8.4.31", + "sass": "^1.77.5", + "@svgr/webpack": "^8.1.0", + "protobufjs-cli": "^1.1.2" + }, + "packageManager": "yarn@1.22.22+sha1.ac34549e6aa8e7ead463a7407e1c7390f61a6610" +} \ No newline at end of file diff --git a/demo/postcss.config.js b/demo/postcss.config.js new file mode 100644 index 00000000..ea748d4d --- /dev/null +++ b/demo/postcss.config.js @@ -0,0 +1,10 @@ +module.exports = { + plugins: { + autoprefixer: {}, + "@minko-fe/postcss-pxtoviewport": { + viewportWidth: 375, + exclude: /node_modules/, + include: /\/src\/platform\/mobile\//, + } + }, +} diff --git a/playground/src/app/api/agents/start/graph.tsx b/demo/src/app/api/agents/start/graph.tsx similarity index 100% rename from playground/src/app/api/agents/start/graph.tsx rename to demo/src/app/api/agents/start/graph.tsx diff --git a/demo/src/app/api/agents/start/route.tsx b/demo/src/app/api/agents/start/route.tsx new file mode 100644 index 00000000..5a7b4440 --- /dev/null +++ b/demo/src/app/api/agents/start/route.tsx @@ -0,0 +1,56 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getGraphProperties } from './graph'; + +/** + * Handles the POST request to start an agent. + * + * @param request - The NextRequest object representing the incoming request. + * @returns A NextResponse object representing the response to be sent back to the client. + */ +export async function POST(request: NextRequest) { + try { + const { AGENT_SERVER_URL } = process.env; + + // Check if environment variables are available + if (!AGENT_SERVER_URL) { + throw "Environment variables are not available"; + } + + const body = await request.json(); + const { + request_id, + channel_name, + user_uid, + graph_name, + language, + voice_type, + } = body; + + // Send a POST request to start the agent + const response = await fetch(`${AGENT_SERVER_URL}/start`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + request_id, + channel_name, + user_uid, + graph_name, + // Get the graph properties based on the graph name, language, and voice type + properties: getGraphProperties(graph_name, language, voice_type), + }), + }); + + const responseData = await response.json(); + + return NextResponse.json(responseData, { status: response.status }); + } catch (error) { + if (error instanceof Response) { + const errorData = await error.json(); + return NextResponse.json(errorData, { status: error.status }); + } else { + return NextResponse.json({ code: "1", data: null, msg: "Internal Server Error" }, { status: 500 }); + } + } +} \ No newline at end of file diff --git a/demo/src/app/favicon.ico b/demo/src/app/favicon.ico new file mode 100644 index 00000000..21b38b96 Binary files /dev/null and b/demo/src/app/favicon.ico differ diff --git a/demo/src/app/global.css b/demo/src/app/global.css new file mode 100644 index 00000000..f7007287 --- /dev/null +++ b/demo/src/app/global.css @@ -0,0 +1,66 @@ +* { + box-sizing: border-box; + padding: 0; + margin: 0; +} + +html, +body { + background-color: #0F0F11; + font-family: "PingFang SC"; +} + +a { + color: inherit; + text-decoration: none; +} + +@media (prefers-color-scheme: dark) { + html { + color-scheme: dark; + } +} + +.ant-select-arrow { + color: #667085 !important; +} + + +.ant-select-selection-item { + color: #667085 !important; +} + +.ant-select-selector { + border: 1px solid #272A2F !important; + background-color: #272A2F !important; +} + +.ant-select-dropdown { + background-color: #1E2024 !important; +} + +.ant-select-item { + background: #1E2024 !important; + color: var(--Grey-600, #667085) !important; +} + +.ant-select-item-option-selected { + background: #272A2F !important; + color: var(--Grey-300, #EAECF0) !important; +} + + +.ant-popover-inner { + /* width: 260px !important; */ + background: #1E2025 !important; +} + + +.ant-select-selection-placeholder { + color: var(--Grey-600, #667085) !important; +} + + +.ant-empty-description { + color: var(--Grey-600, #667085) !important; +} diff --git a/playground/src/app/home/page.tsx b/demo/src/app/home/page.tsx similarity index 100% rename from playground/src/app/home/page.tsx rename to demo/src/app/home/page.tsx diff --git a/demo/src/app/index.module.scss b/demo/src/app/index.module.scss new file mode 100644 index 00000000..78585596 --- /dev/null +++ b/demo/src/app/index.module.scss @@ -0,0 +1,93 @@ +@function multiple-box-shadow($n) { + $value: '#{random(2000)}px #{random(2000)}px #FFF'; + + @for $i from 2 through $n { + $value: '#{$value}, #{random(2000)}px #{random(2000)}px #FFF'; + } + + @return unquote($value); +} + +$shadows-small: multiple-box-shadow(700); +$shadows-medium: multiple-box-shadow(200); +$shadows-big: multiple-box-shadow(100); + + +.login { + position: absolute; + left: 0; + top: 0; + width: 100%; + height: 100%; + overflow: hidden; + // background: radial-gradient(ellipse at bottom, #1B2735 0%, #090A0F 100%); + background: url("../assets/background.jpg") no-repeat center center; + background-size: cover; + box-sizing: border-box; + + .starts { + width: 1px; + height: 1px; + background: transparent; + box-shadow: $shadows-small; + animation: animStar 50s linear infinite; + + &:after { + content: " "; + position: absolute; + top: 2000px; + width: 1px; + height: 1px; + background: transparent; + box-shadow: $shadows-small + } + } + + .starts2 { + width: 2px; + height: 2px; + box-shadow: $shadows-medium; + animation: animStar 100s linear infinite; + + &:after { + content: " "; + position: absolute; + top: 2000px; + width: 2px; + height: 2px; + background: transparent; + box-shadow: $shadows-medium; + } + } + + .starts3 { + width: 3px; + height: 3px; + background: transparent; + box-shadow: $shadows-big; + animation: animStar 150s linear infinite; + + &:after { + content: " "; + position: absolute; + top: 2000px; + width: 3px; + height: 3px; + background: transparent; + box-shadow: $shadows-big; + } + + } + +} + + +@keyframes animStar { + from { + transform: translateY(0px) + } + + to { + transform: translateY(-2000px) + } +} diff --git a/demo/src/app/layout.tsx b/demo/src/app/layout.tsx new file mode 100644 index 00000000..b6153573 --- /dev/null +++ b/demo/src/app/layout.tsx @@ -0,0 +1,53 @@ +import { ConfigProvider } from "antd" +import { StoreProvider } from "@/store"; +import type { Metadata, Viewport } from "next"; + +import './global.css' + + +export const metadata: Metadata = { + title: "Astra.ai", + description: "A multimodal agent powered by TEN", + appleWebApp: { + capable: true, + statusBarStyle: "black", + } +}; + +export const viewport: Viewport = { + width: "device-width", + initialScale: 1, + minimumScale: 1, + maximumScale: 1, + userScalable: false, + viewportFit: "cover", +} + + + + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + + + {children} + + + + + ); +} diff --git a/demo/src/app/page.tsx b/demo/src/app/page.tsx new file mode 100644 index 00000000..1bdcdeda --- /dev/null +++ b/demo/src/app/page.tsx @@ -0,0 +1,14 @@ +import LoginCard from "@/components/loginCard" +import styles from "./index.module.scss" + +export default function Login() { + + return ( +
+
+
+
+ +
+ ); +} diff --git a/demo/src/assets/background.jpg b/demo/src/assets/background.jpg new file mode 100644 index 00000000..02762343 Binary files /dev/null and b/demo/src/assets/background.jpg differ diff --git a/demo/src/assets/cam_mute.svg b/demo/src/assets/cam_mute.svg new file mode 100644 index 00000000..a4640ae4 --- /dev/null +++ b/demo/src/assets/cam_mute.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/cam_unmute.svg b/demo/src/assets/cam_unmute.svg new file mode 100644 index 00000000..1eebfaa6 --- /dev/null +++ b/demo/src/assets/cam_unmute.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/color_picker.svg b/demo/src/assets/color_picker.svg new file mode 100644 index 00000000..fb9bb33e --- /dev/null +++ b/demo/src/assets/color_picker.svg @@ -0,0 +1,17 @@ + + + + + + + + + + diff --git a/demo/src/assets/github.svg b/demo/src/assets/github.svg new file mode 100644 index 00000000..e6566c41 --- /dev/null +++ b/demo/src/assets/github.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/info.svg b/demo/src/assets/info.svg new file mode 100644 index 00000000..8ca99511 --- /dev/null +++ b/demo/src/assets/info.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/logo.svg b/demo/src/assets/logo.svg new file mode 100644 index 00000000..af99893a --- /dev/null +++ b/demo/src/assets/logo.svg @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demo/src/assets/logo_small.svg b/demo/src/assets/logo_small.svg new file mode 100644 index 00000000..34e755bd --- /dev/null +++ b/demo/src/assets/logo_small.svg @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demo/src/assets/mic_mute.svg b/demo/src/assets/mic_mute.svg new file mode 100644 index 00000000..dd4a17dd --- /dev/null +++ b/demo/src/assets/mic_mute.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/mic_unmute.svg b/demo/src/assets/mic_unmute.svg new file mode 100644 index 00000000..18e78236 --- /dev/null +++ b/demo/src/assets/mic_unmute.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/network/average.svg b/demo/src/assets/network/average.svg new file mode 100644 index 00000000..9a27072f --- /dev/null +++ b/demo/src/assets/network/average.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/demo/src/assets/network/disconnected.svg b/demo/src/assets/network/disconnected.svg new file mode 100644 index 00000000..b7db1d71 --- /dev/null +++ b/demo/src/assets/network/disconnected.svg @@ -0,0 +1,9 @@ + + + + + diff --git a/demo/src/assets/network/excellent.svg b/demo/src/assets/network/excellent.svg new file mode 100644 index 00000000..55b9fc9e --- /dev/null +++ b/demo/src/assets/network/excellent.svg @@ -0,0 +1,6 @@ + + + + diff --git a/demo/src/assets/network/good.svg b/demo/src/assets/network/good.svg new file mode 100644 index 00000000..8c36a7e7 --- /dev/null +++ b/demo/src/assets/network/good.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/demo/src/assets/network/poor.svg b/demo/src/assets/network/poor.svg new file mode 100644 index 00000000..d9df0238 --- /dev/null +++ b/demo/src/assets/network/poor.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/demo/src/assets/pdf.svg b/demo/src/assets/pdf.svg new file mode 100644 index 00000000..dc67f4d5 --- /dev/null +++ b/demo/src/assets/pdf.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/assets/transcription.svg b/demo/src/assets/transcription.svg new file mode 100644 index 00000000..8b887a6f --- /dev/null +++ b/demo/src/assets/transcription.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/demo/src/assets/voice.svg b/demo/src/assets/voice.svg new file mode 100644 index 00000000..86a880b0 --- /dev/null +++ b/demo/src/assets/voice.svg @@ -0,0 +1,3 @@ + + + diff --git a/demo/src/common/constant.ts b/demo/src/common/constant.ts new file mode 100644 index 00000000..fee0c18e --- /dev/null +++ b/demo/src/common/constant.ts @@ -0,0 +1,88 @@ +import { IOptions, ColorItem, LanguageOptionItem, VoiceOptionItem, GraphOptionItem } from "@/types" +export const GITHUB_URL = "https://github.com/TEN-framework/ASTRA.ai" +export const OPTIONS_KEY = "__options__" +export const DEFAULT_OPTIONS: IOptions = { + channel: "", + userName: "", + userId: 0 +} +export const DESCRIPTION = "This is an AI voice assistant powered by ASTRA.ai framework, Agora, Azure and ChatGPT." +export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [ + { + label: "English", + value: "en-US" + }, + { + label: "Chinese", + value: "zh-CN" + }, + { + label: "Korean", + value: "ko-KR" + }, + { + label: "Japanese", + value: "ja-JP" + } +] +export const GRAPH_OPTIONS: GraphOptionItem[] = [ + { + label: "Voice Agent - OpenAI LLM + Azure TTS", + value: "va.openai.azure" + }, + { + label: "Voice Agent with Vision - OpenAI LLM + Azure TTS", + value: "camera.va.openai.azure" + }, + { + label: "Voice Agent with Knowledge - RAG + Qwen LLM + Cosy TTS", + value: "va.qwen.rag" + }, +] + +export const isRagGraph = (graphName: string) => { + return graphName === "va.qwen.rag" +} + +export const VOICE_OPTIONS: VoiceOptionItem[] = [ + { + label: "Male", + value: "male" + }, + { + label: "Female", + value: "female" + } +] +export const COLOR_LIST: ColorItem[] = [{ + active: "#0888FF", + default: "#143354" +}, { + active: "#563FD8", + default: "#2C2553" +}, +{ + active: "#18A957", + default: "#173526" +}, { + active: "#FFAB08", + default: "#423115" +}, { + active: "#FD5C63", + default: "#462629" +}, { + active: "#E225B2", + default: "#481C3F" +}] + +export type VoiceTypeMap = { + [voiceType: string]: string; +}; + +export type VendorNameMap = { + [vendorName: string]: VoiceTypeMap; +}; + +export type LanguageMap = { + [language: string]: VendorNameMap; +}; \ No newline at end of file diff --git a/demo/src/common/hooks.ts b/demo/src/common/hooks.ts new file mode 100644 index 00000000..9759fa29 --- /dev/null +++ b/demo/src/common/hooks.ts @@ -0,0 +1,131 @@ +"use client" + +import { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" +import { normalizeFrequencies } from "./utils" +import { useState, useEffect, useMemo, useRef } from "react" +import type { AppDispatch, AppStore, RootState } from "../store" +import { useDispatch, useSelector, useStore } from "react-redux" +import { Grid } from "antd" + +const { useBreakpoint } = Grid; + +export const useAppDispatch = useDispatch.withTypes() +export const useAppSelector = useSelector.withTypes() +export const useAppStore = useStore.withTypes() + +export const useMultibandTrackVolume = ( + track?: IMicrophoneAudioTrack | MediaStreamTrack, + bands: number = 5, + loPass: number = 100, + hiPass: number = 600 +) => { + const [frequencyBands, setFrequencyBands] = useState([]); + + useEffect(() => { + if (!track) { + return setFrequencyBands(new Array(bands).fill(new Float32Array(0))) + } + + const ctx = new AudioContext(); + let finTrack = track instanceof MediaStreamTrack ? track : track.getMediaStreamTrack() + const mediaStream = new MediaStream([finTrack]); + const source = ctx.createMediaStreamSource(mediaStream); + const analyser = ctx.createAnalyser(); + analyser.fftSize = 2048 + + source.connect(analyser); + + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Float32Array(bufferLength); + + const updateVolume = () => { + analyser.getFloatFrequencyData(dataArray); + let frequencies: Float32Array = new Float32Array(dataArray.length); + for (let i = 0; i < dataArray.length; i++) { + frequencies[i] = dataArray[i]; + } + frequencies = frequencies.slice(loPass, hiPass); + + const normalizedFrequencies = normalizeFrequencies(frequencies); + const chunkSize = Math.ceil(normalizedFrequencies.length / bands); + const chunks: Float32Array[] = []; + for (let i = 0; i < bands; i++) { + chunks.push( + normalizedFrequencies.slice(i * chunkSize, (i + 1) * chunkSize) + ); + } + + setFrequencyBands(chunks); + }; + + const interval = setInterval(updateVolume, 10); + + return () => { + source.disconnect(); + clearInterval(interval); + }; + }, [track, loPass, hiPass, bands]); + + return frequencyBands; +}; + +export const useAutoScroll = (ref: React.RefObject) => { + + const callback: MutationCallback = (mutationList, observer) => { + mutationList.forEach((mutation) => { + switch (mutation.type) { + case "childList": + if (!ref.current) { + return + } + ref.current.scrollTop = ref.current.scrollHeight; + break; + } + }) + } + + useEffect(() => { + if (!ref.current) { + return; + } + const observer = new MutationObserver(callback); + observer.observe(ref.current, { + childList: true, + subtree: true + }); + + return () => { + observer.disconnect(); + }; + }, [ref]); +} + +export const useSmallScreen = () => { + const screens = useBreakpoint(); + + const xs = useMemo(() => { + return !screens.sm && screens.xs + }, [screens]) + + const sm = useMemo(() => { + return !screens.md && screens.sm + }, [screens]) + + return { + xs, + sm, + isSmallScreen: xs || sm + } +} + +export const usePrevious = (value: any) => { + const ref = useRef(); + + useEffect(() => { + ref.current = value; + }, [value]); + + return ref.current; +}; + + diff --git a/demo/src/common/index.ts b/demo/src/common/index.ts new file mode 100644 index 00000000..3c2b0300 --- /dev/null +++ b/demo/src/common/index.ts @@ -0,0 +1,6 @@ +export * from "./hooks" +export * from "./constant" +export * from "./utils" +export * from "./storage" +export * from "./request" +export * from "./mock" diff --git a/demo/src/common/mock.ts b/demo/src/common/mock.ts new file mode 100644 index 00000000..db1e2ff8 --- /dev/null +++ b/demo/src/common/mock.ts @@ -0,0 +1,41 @@ +import { getRandomUserId } from "./utils" +import { IChatItem } from "@/types" + + +const SENTENCES = [ + "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium.", + "Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit.", + "Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit.", + "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.", + "Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.", + "Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", +] + + +export const genRandomParagraph = (num: number = 0): string => { + let paragraph = "" + for (let i = 0; i < num; i++) { + const randomIndex = Math.floor(Math.random() * SENTENCES.length) + paragraph += SENTENCES[randomIndex] + " " + } + + return paragraph.trim() +} + + +export const genRandomChatList = (num: number = 10): IChatItem[] => { + const arr: IChatItem[] = [] + for (let i = 0; i < num; i++) { + const type = Math.random() > 0.5 ? "agent" : "user" + arr.push({ + userId: getRandomUserId(), + userName: type == "agent" ? "Agent" : "You", + text: genRandomParagraph(3), + type, + time: Date.now(), + }) + } + + return arr +} diff --git a/demo/src/common/request.ts b/demo/src/common/request.ts new file mode 100644 index 00000000..160cc065 --- /dev/null +++ b/demo/src/common/request.ts @@ -0,0 +1,133 @@ +import { genUUID } from "./utils" +import { Language } from "@/types" + +interface StartRequestConfig { + channel: string + userId: number, + graphName: string, + language: Language, + voiceType: "male" | "female" +} + +interface GenAgoraDataConfig { + userId: string | number + channel: string +} + +export const apiGenAgoraData = async (config: GenAgoraDataConfig) => { + // the request will be rewrite at next.config.mjs to send to $AGENT_SERVER_URL + const url = `/api/token/generate` + const { userId, channel } = config + const data = { + request_id: genUUID(), + uid: userId, + channel_name: channel + } + let resp: any = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(data), + }) + resp = (await resp.json()) || {} + return resp +} + +export const apiStartService = async (config: StartRequestConfig): Promise => { + // look at app/api/agents/start/route.tsx for the server-side implementation + const url = `/api/agents/start` + const { channel, userId, graphName, language, voiceType } = config + const data = { + request_id: genUUID(), + channel_name: channel, + user_uid: userId, + graph_name: graphName, + language, + voice_type: voiceType + } + let resp: any = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(data), + }) + resp = (await resp.json()) || {} + return resp +} + +export const apiStopService = async (channel: string) => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/agents/stop` + const data = { + request_id: genUUID(), + channel_name: channel + } + let resp = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(data), + }) + resp = (await resp.json()) || {} + return resp +} + +export const apiGetDocumentList = async () => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/vector/document/preset/list` + let resp: any = await fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }) + resp = (await resp.json()) || {} + if (resp.code !== "0") { + throw new Error(resp.msg) + } + return resp +} + +export const apiUpdateDocument = async (options: { channel: string, collection: string, fileName: string }) => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/vector/document/update` + const { channel, collection, fileName } = options + const data = { + request_id: genUUID(), + channel_name: channel, + collection: collection, + file_name: fileName + } + let resp: any = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(data), + }) + resp = (await resp.json()) || {} + return resp +} + + +// ping/pong +export const apiPing = async (channel: string) => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/agents/ping` + const data = { + request_id: genUUID(), + channel_name: channel + } + let resp = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(data), + }) + resp = (await resp.json()) || {} + return resp +} diff --git a/demo/src/common/storage.ts b/demo/src/common/storage.ts new file mode 100644 index 00000000..ed96083d --- /dev/null +++ b/demo/src/common/storage.ts @@ -0,0 +1,21 @@ +import { IOptions } from "@/types" +import { OPTIONS_KEY, DEFAULT_OPTIONS } from "./constant" + +export const getOptionsFromLocal = () => { + if (typeof window !== "undefined") { + const data = localStorage.getItem(OPTIONS_KEY) + if (data) { + return JSON.parse(data) + } + } + return DEFAULT_OPTIONS +} + + +export const setOptionsToLocal = (options: IOptions) => { + if (typeof window !== "undefined") { + localStorage.setItem(OPTIONS_KEY, JSON.stringify(options)) + } +} + + diff --git a/demo/src/common/utils.ts b/demo/src/common/utils.ts new file mode 100644 index 00000000..1d6f0d00 --- /dev/null +++ b/demo/src/common/utils.ts @@ -0,0 +1,59 @@ +export const genRandomString = (length: number = 10) => { + let result = ''; + const characters = 'abcdefghijklmnopqrstuvwxyz0123456789'; + const charactersLength = characters.length; + + for (let i = 0; i < length; i++) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + } + + return result; +} + + +export const getRandomUserId = (): number => { + return Math.floor(Math.random() * 99999) + 100000 +} + +export const getRandomChannel = (number = 6) => { + return "agora_" + genRandomString(number) +} + + +export const sleep = (ms: number) => { + return new Promise(resolve => setTimeout(resolve, ms)); +} + + +export const normalizeFrequencies = (frequencies: Float32Array) => { + const normalizeDb = (value: number) => { + const minDb = -100; + const maxDb = -10; + let db = 1 - (Math.max(minDb, Math.min(maxDb, value)) * -1) / 100; + db = Math.sqrt(db); + + return db; + }; + + // Normalize all frequency values + return frequencies.map((value) => { + if (value === -Infinity) { + return 0; + } + return normalizeDb(value); + }); +}; + + +export const genUUID = () => { + return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function (c) { + const r = (Math.random() * 16) | 0 + const v = c === "x" ? r : (r & 0x3) | 0x8 + return v.toString(16) + }) +} + + +export const isMobile = () => { + return /Mobile|iPhone|iPad|Android|Windows Phone/i.test(navigator.userAgent) +} \ No newline at end of file diff --git a/demo/src/components/authInitializer/index.tsx b/demo/src/components/authInitializer/index.tsx new file mode 100644 index 00000000..5ef763a1 --- /dev/null +++ b/demo/src/components/authInitializer/index.tsx @@ -0,0 +1,29 @@ +"use client" + +import { ReactNode, useEffect } from "react" +import { useAppDispatch, getOptionsFromLocal } from "@/common" +import { setOptions, reset } from "@/store/reducers/global" + +interface AuthInitializerProps { + children: ReactNode; +} + +const AuthInitializer = (props: AuthInitializerProps) => { + const { children } = props; + const dispatch = useAppDispatch() + + useEffect(() => { + if (typeof window !== "undefined") { + const options = getOptionsFromLocal() + if (options) { + dispatch(reset()) + dispatch(setOptions(options)) + } + } + }, [dispatch]) + + return children +} + + +export default AuthInitializer; diff --git a/demo/src/components/customSelect/index.module.scss b/demo/src/components/customSelect/index.module.scss new file mode 100644 index 00000000..0649e994 --- /dev/null +++ b/demo/src/components/customSelect/index.module.scss @@ -0,0 +1,22 @@ +.selectWrapper { + position: relative; + + .prefixIconWrapper { + position: absolute; + z-index: 1; + width: 3rem; + height: 100%; + display: flex; + align-items: center; + justify-content: center; + } + + :global(.customSelect) { + width: 100%; + + :global(.ant-select-selector) { + padding-left: calc(3rem - 8px) !important; + } + } + +} diff --git a/demo/src/components/customSelect/index.tsx b/demo/src/components/customSelect/index.tsx new file mode 100644 index 00000000..8dd1b188 --- /dev/null +++ b/demo/src/components/customSelect/index.tsx @@ -0,0 +1,19 @@ +import { Select, SelectProps } from "antd" +import styles from "./index.module.scss" + +type CustomSelectProps = SelectProps & { + prefixIcon?: React.ReactNode; +} + +const CustomSelect = (props: CustomSelectProps) => { + + const { prefixIcon, className, ...rest } = props; + + return
+ {prefixIcon &&
{prefixIcon}
} + +
+} + + +export default CustomSelect diff --git a/demo/src/components/icons/cam/index.tsx b/demo/src/components/icons/cam/index.tsx new file mode 100644 index 00000000..628e651c --- /dev/null +++ b/demo/src/components/icons/cam/index.tsx @@ -0,0 +1,17 @@ +import camMuteSvg from "@/assets/cam_mute.svg" +import camUnMuteSvg from "@/assets/cam_unmute.svg" +import { IconProps } from "../types" + +interface ICamIconProps extends IconProps { + active?: boolean +} + +export const CamIcon = (props: ICamIconProps) => { + const { active, ...rest } = props + + if (active) { + return camUnMuteSvg(rest) + } else { + return camMuteSvg(rest) + } +} diff --git a/demo/src/components/icons/colorPicker/index.tsx b/demo/src/components/icons/colorPicker/index.tsx new file mode 100644 index 00000000..81efcb12 --- /dev/null +++ b/demo/src/components/icons/colorPicker/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import ColorPickerSvg from "@/assets/color_picker.svg" + +export const ColorPickerIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/github/index.tsx b/demo/src/components/icons/github/index.tsx new file mode 100644 index 00000000..cee01b8b --- /dev/null +++ b/demo/src/components/icons/github/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import GithubSvg from "@/assets/github.svg" + +export const GithubIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/index.tsx b/demo/src/components/icons/index.tsx new file mode 100644 index 00000000..e303674a --- /dev/null +++ b/demo/src/components/icons/index.tsx @@ -0,0 +1,10 @@ +export * from "./mic" +export * from "./cam" +export * from "./network" +export * from "./github" +export * from "./transcription" +export * from "./logo" +export * from "./info" +export * from "./colorPicker" +export * from "./voice" +export * from "./pdf" diff --git a/demo/src/components/icons/info/index.tsx b/demo/src/components/icons/info/index.tsx new file mode 100644 index 00000000..cf783be9 --- /dev/null +++ b/demo/src/components/icons/info/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import InfoSvg from "@/assets/info.svg" + +export const InfoIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/logo/index.tsx b/demo/src/components/icons/logo/index.tsx new file mode 100644 index 00000000..f86d5246 --- /dev/null +++ b/demo/src/components/icons/logo/index.tsx @@ -0,0 +1,8 @@ +import { IconProps } from "../types" +import LogoSvg from "@/assets/logo.svg" +import SmallLogoSvg from "@/assets/logo_small.svg" + +export const LogoIcon = (props: IconProps) => { + const { size = "default" } = props + return size == "small" ? : +} diff --git a/demo/src/components/icons/mic/index.tsx b/demo/src/components/icons/mic/index.tsx new file mode 100644 index 00000000..0a693033 --- /dev/null +++ b/demo/src/components/icons/mic/index.tsx @@ -0,0 +1,23 @@ +import { IconProps } from "../types" +import micMuteSvg from "@/assets/mic_mute.svg" +import micUnMuteSvg from "@/assets/mic_unmute.svg" + +interface IMicIconProps extends IconProps { + active?: boolean +} + +export const MicIcon = (props: IMicIconProps) => { + const { active, color, ...rest } = props + + if (active) { + return micUnMuteSvg({ + color: color || "#3D53F5", + ...rest, + }) + } else { + return micMuteSvg({ + color: color || "#667085", + ...rest, + }) + } +} diff --git a/demo/src/components/icons/network/index.tsx b/demo/src/components/icons/network/index.tsx new file mode 100644 index 00000000..1950cda7 --- /dev/null +++ b/demo/src/components/icons/network/index.tsx @@ -0,0 +1,33 @@ +import averageSvg from "@/assets/network/average.svg" +import goodSvg from "@/assets/network/good.svg" +import poorSvg from "@/assets/network/poor.svg" +import disconnectedSvg from "@/assets/network/disconnected.svg" +import excellentSvg from "@/assets/network/excellent.svg" + +import { IconProps } from "../types" + +interface INetworkIconProps extends IconProps { + level?: number +} + +export const NetworkIcon = (props: INetworkIconProps) => { + const { level, ...rest } = props + switch (level) { + case 0: + return disconnectedSvg(rest) + case 1: + return excellentSvg(rest) + case 2: + return goodSvg(rest) + case 3: + return averageSvg(rest) + case 4: + return averageSvg(rest) + case 5: + return poorSvg(rest) + case 6: + return disconnectedSvg(rest) + default: + return disconnectedSvg(rest) + } +} diff --git a/demo/src/components/icons/pdf/index.tsx b/demo/src/components/icons/pdf/index.tsx new file mode 100644 index 00000000..83de8b2d --- /dev/null +++ b/demo/src/components/icons/pdf/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import PdfSvg from "@/assets/pdf.svg" + +export const PdfIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/transcription/index.tsx b/demo/src/components/icons/transcription/index.tsx new file mode 100644 index 00000000..757adbce --- /dev/null +++ b/demo/src/components/icons/transcription/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import TranscriptionSvg from "@/assets/transcription.svg" + +export const TranscriptionIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/icons/types.ts b/demo/src/components/icons/types.ts new file mode 100644 index 00000000..c37e8133 --- /dev/null +++ b/demo/src/components/icons/types.ts @@ -0,0 +1,10 @@ +export interface IconProps { + width?: number + height?: number + color?: string + viewBox?: string + size?: "small" | "default" + // style?: React.CSSProperties + transform?: string + onClick?: () => void +} diff --git a/demo/src/components/icons/voice/index.tsx b/demo/src/components/icons/voice/index.tsx new file mode 100644 index 00000000..87164cea --- /dev/null +++ b/demo/src/components/icons/voice/index.tsx @@ -0,0 +1,6 @@ +import { IconProps } from "../types" +import VoiceSvg from "@/assets/voice.svg" + +export const VoiceIcon = (props: IconProps) => { + return +} diff --git a/demo/src/components/loginCard/index.module.scss b/demo/src/components/loginCard/index.module.scss new file mode 100644 index 00000000..966ebc20 --- /dev/null +++ b/demo/src/components/loginCard/index.module.scss @@ -0,0 +1,112 @@ +.card { + position: absolute; + top: 45%; + left: 50%; + transform: translate(-50%, -50%); + display: flex; + width: 368px; + padding: 100px 24px 40px 24px; + flex-direction: column; + justify-content: center; + align-items: center; + border-radius: 20px; + border: 1px solid #20272D; + background: linear-gradient(154deg, rgba(31, 69, 141, 0.16) 0%, rgba(31, 69, 141, 0.00) 30%), linear-gradient(153deg, rgba(31, 54, 97, 0.00) 53.75%, #1F458D 100%), rgba(15, 15, 17, 0.10); + box-shadow: 0px 3.999px 48.988px 0px rgba(0, 7, 72, 0.12); + backdrop-filter: blur(8.8px); + + .top { + .github { + position: absolute; + right: 24px; + top: 24px; + display: flex; + padding: 4px 8px 4px 4px; + align-items: center; + gap: 4px; + border-radius: 100px; + border: 1px solid #2B2F36; + cursor: pointer; + + .text { + color: var(--Grey-300, #EAECF0); + font-size: 12px; + line-height: 150%; + } + } + } + + .content { + + .title { + margin-bottom: 32px; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 12px; + + .text { + margin-top: 8px; + color: var(--Grey-300, #EAECF0); + text-align: center; + font-size: 18px; + font-weight: 500; + } + } + + .section { + + input { + display: flex; + width: 320px; + flex-direction: column; + align-items: flex-start; + gap: 6px; + display: flex; + height: 38px; + padding: 12px 8px; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 6px; + border: 1px solid #2B2F36; + box-shadow: 0px 4.282px 52.456px 0px rgba(0, 7, 72, 0.12); + backdrop-filter: blur(13px); + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.20); + } + + .btn { + display: flex; + padding: 10px 18px; + justify-content: center; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 8px; + background: var(--primary-500-base, #0888FF); + box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05); + cursor: pointer; + + .btnText { + color: var(---white, #FFF); + font-size: 16px; + font-weight: 500; + line-height: 24px; + } + } + } + + .section+.section { + margin-top: 24px; + } + + .version { + text-align: center; + margin-top: 32px; + color: var(--Grey-600, #667085); + line-height: 22px; + } + + } +} diff --git a/demo/src/components/loginCard/index.tsx b/demo/src/components/loginCard/index.tsx new file mode 100644 index 00000000..56a78986 --- /dev/null +++ b/demo/src/components/loginCard/index.tsx @@ -0,0 +1,77 @@ +"use client" + +import packageData from "../../../package.json" +import { useRouter } from 'next/navigation' +import { message } from "antd" +import { useState } from "react" +import { GithubIcon, LogoIcon } from "../icons" +import { GITHUB_URL, getRandomUserId, useAppDispatch, getRandomChannel } from "@/common" +import { setOptions } from "@/store/reducers/global" +import styles from "./index.module.scss" + + +const { version } = packageData + +const LoginCard = () => { + const dispatch = useAppDispatch() + const router = useRouter() + const [userName, setUserName] = useState("") + + const onClickGithub = () => { + if (typeof window !== "undefined") { + window.open(GITHUB_URL, "_blank") + } + } + + const onUserNameChange = (e: any) => { + let value = e.target.value + value = value.replace(/\s/g, ""); + setUserName(value) + } + + + + const onClickJoin = () => { + if (!userName) { + message.error("please input user name") + return + } + const userId = getRandomUserId() + dispatch(setOptions({ + userName, + channel: getRandomChannel(), + userId + })) + router.push("/home") + } + + + return
+
+ + + GitHub + +
+
+
+ + Astra - a multimodal interactive agent +
+
+ +
+
+
+ Join +
+
+
Version {version}
+
+
+ + + return +} + +export default LoginCard diff --git a/demo/src/components/pdfSelect/index.module.scss b/demo/src/components/pdfSelect/index.module.scss new file mode 100644 index 00000000..adb93280 --- /dev/null +++ b/demo/src/components/pdfSelect/index.module.scss @@ -0,0 +1,8 @@ +// .pdfSelect { + // min-width: 200px; + // max-width: 300px; + // } +.dropdownRender { + display: flex; + justify-content: flex-end; +} diff --git a/demo/src/components/pdfSelect/index.tsx b/demo/src/components/pdfSelect/index.tsx new file mode 100644 index 00000000..f593bf1d --- /dev/null +++ b/demo/src/components/pdfSelect/index.tsx @@ -0,0 +1,88 @@ +import { ReactElement, useState } from "react" +import { PdfIcon } from "@/components/icons" +import CustomSelect from "@/components/customSelect" +import { Divider, message } from 'antd'; +import { useEffect } from 'react'; +import { apiGetDocumentList, apiUpdateDocument, useAppSelector } from "@/common" +import PdfUpload from "./upload" +import { OptionType, IPdfData } from "@/types" + +import styles from "./index.module.scss" + +const PdfSelect = () => { + const options = useAppSelector(state => state.global.options) + const { channel } = options + const [pdfOptions, setPdfOptions] = useState([]) + const [selectedPdf, setSelectedPdf] = useState('') + const agentConnected = useAppSelector(state => state.global.agentConnected) + + + useEffect(() => { + if(agentConnected) { + getPDFOptions() + } else { + setPdfOptions([{ + value: '', + label: 'Please select a PDF file' + }]) + } + }, [agentConnected]) + + + const getPDFOptions = async () => { + const res = await apiGetDocumentList() + setPdfOptions([{ + value: '', + label: 'Please select a PDF file' + }].concat(res.data.map((item: any) => { + return { + value: item.collection, + label: item.file_name + } + }))) + setSelectedPdf('') + } + + const onUploadSuccess = (data: IPdfData) => { + setPdfOptions([...pdfOptions, { + value: data.collection, + label: data.fileName + }]) + setSelectedPdf(data.collection) + } + + const pdfDropdownRender = (menu: ReactElement) => { + return <> + {menu} + +
+ +
+ + } + + + const onSelectPdf = async (val: string) => { + const item = pdfOptions.find(item => item.value === val) + if (!item) { + return message.error("Please select a PDF file") + } + setSelectedPdf(val) + await apiUpdateDocument({ + collection: val, + fileName: item.label, + channel + }) + } + + + return } + onChange={onSelectPdf} + value={selectedPdf} + options={pdfOptions} + dropdownRender={pdfDropdownRender} + className={styles.pdfSelect} placeholder="Select a PDF file"> +} + +export default PdfSelect diff --git a/demo/src/components/pdfSelect/upload/index.module.scss b/demo/src/components/pdfSelect/upload/index.module.scss new file mode 100644 index 00000000..fd559b5e --- /dev/null +++ b/demo/src/components/pdfSelect/upload/index.module.scss @@ -0,0 +1,7 @@ +.btn { + color: var(--theme-color, #EAECF0); + + &:hover { + color: var(--theme-color, #EAECF0) !important; + } +} diff --git a/demo/src/components/pdfSelect/upload/index.tsx b/demo/src/components/pdfSelect/upload/index.tsx new file mode 100644 index 00000000..4eab9684 --- /dev/null +++ b/demo/src/components/pdfSelect/upload/index.tsx @@ -0,0 +1,75 @@ +import { Select, Button, message, Upload, UploadProps } from "antd" +import { useState } from "react" +import { PlusOutlined, LoadingOutlined } from '@ant-design/icons'; +import { useAppSelector, genUUID } from "@/common" +import { IPdfData } from "@/types" + +import styles from "./index.module.scss" + +interface PdfSelectProps { + onSuccess?: (data: IPdfData) => void +} + +const PdfUpload = (props: PdfSelectProps) => { + const { onSuccess } = props + const agentConnected = useAppSelector(state => state.global.agentConnected) + const options = useAppSelector(state => state.global.options) + const { channel, userId } = options + + const [uploading, setUploading] = useState(false) + + const uploadProps: UploadProps = { + accept: "application/pdf", + maxCount: 1, + showUploadList: false, + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + action: `/api/vector/document/upload`, + data: { + channel_name: channel, + uid: String(userId), + request_id: genUUID() + }, + onChange: (info) => { + const { file } = info + const { status, name } = file + if (status == "uploading") { + setUploading(true) + } else if (status == 'done') { + setUploading(false) + const { response } = file + if (response.code == "0") { + message.success(`Upload ${name} success`) + const { collection, file_name } = response.data + onSuccess && onSuccess({ + fileName: file_name, + collection + }) + } else { + message.error(response.msg) + } + } else if (status == 'error') { + setUploading(false) + message.error(`Upload ${name} failed`) + } + } + } + + const onClickUploadPDF = (e: any) => { + if (!agentConnected) { + message.error("Please connect to agent first") + e.stopPropagation() + } + } + + + return + + +} + + +export default PdfUpload diff --git a/demo/src/manager/events.ts b/demo/src/manager/events.ts new file mode 100644 index 00000000..055c6d87 --- /dev/null +++ b/demo/src/manager/events.ts @@ -0,0 +1,51 @@ +import { EventHandler } from "./types" + +export class AGEventEmitter { + private readonly _eventMap: Map[]> = new Map() + + once(evt: Key, cb: T[Key]) { + const wrapper = (...args: any[]) => { + this.off(evt, wrapper as any) + ;(cb as any)(...args) + } + this.on(evt, wrapper as any) + return this + } + + on(evt: Key, cb: T[Key]) { + const cbs = this._eventMap.get(evt) ?? [] + cbs.push(cb as any) + this._eventMap.set(evt, cbs) + return this + } + + off(evt: Key, cb: T[Key]) { + const cbs = this._eventMap.get(evt) + if (cbs) { + this._eventMap.set( + evt, + cbs.filter((it) => it !== cb), + ) + } + return this + } + + removeAllEventListeners(): void { + this._eventMap.clear() + } + + emit(evt: Key, ...args: any[]) { + const cbs = this._eventMap.get(evt) ?? [] + for (const cb of cbs) { + try { + cb && cb(...args) + } catch (e) { + // cb exception should not affect other callbacks + const error = e as Error + const details = error.stack || error.message + console.error(`[event] handling event ${evt.toString()} fail: ${details}`) + } + } + return this + } +} diff --git a/demo/src/manager/index.ts b/demo/src/manager/index.ts new file mode 100644 index 00000000..fa9dfbf2 --- /dev/null +++ b/demo/src/manager/index.ts @@ -0,0 +1 @@ +export * from "./rtc" diff --git a/demo/src/manager/rtc/index.ts b/demo/src/manager/rtc/index.ts new file mode 100644 index 00000000..e9fd6272 --- /dev/null +++ b/demo/src/manager/rtc/index.ts @@ -0,0 +1,2 @@ +export * from "./rtc" +export * from "./types" diff --git a/demo/src/manager/rtc/rtc.ts b/demo/src/manager/rtc/rtc.ts new file mode 100644 index 00000000..4139d89e --- /dev/null +++ b/demo/src/manager/rtc/rtc.ts @@ -0,0 +1,199 @@ +"use client" + +import protoRoot from "@/protobuf/SttMessage_es6.js" +import AgoraRTC, { + IAgoraRTCClient, + IMicrophoneAudioTrack, + IRemoteAudioTrack, + UID, +} from "agora-rtc-sdk-ng" +import { ITextItem } from "@/types" +import { AGEventEmitter } from "../events" +import { RtcEvents, IUserTracks } from "./types" +import { apiGenAgoraData } from "@/common" + +export class RtcManager extends AGEventEmitter { + private _joined + client: IAgoraRTCClient + localTracks: IUserTracks + + constructor() { + super() + this._joined = false + this.localTracks = {} + this.client = AgoraRTC.createClient({ mode: "rtc", codec: "vp8" }) + this._listenRtcEvents() + } + + async join({ channel, userId }: { channel: string; userId: number }) { + if (!this._joined) { + const res = await apiGenAgoraData({ channel, userId }) + const { code, data } = res + if (code != 0) { + throw new Error("Failed to get Agora token") + } + const { appId, token } = data + await this.client?.join(appId, channel, token, userId) + this._joined = true + } + } + + async createTracks() { + try { + const videoTrack = await AgoraRTC.createCameraVideoTrack() + this.localTracks.videoTrack = videoTrack + } catch (err) { + console.error("Failed to create video track", err) + } + try { + const audioTrack = await AgoraRTC.createMicrophoneAudioTrack() + this.localTracks.audioTrack = audioTrack + } catch (err) { + console.error("Failed to create audio track", err) + } + this.emit("localTracksChanged", this.localTracks) + } + + async publish() { + const tracks = [] + if (this.localTracks.videoTrack) { + tracks.push(this.localTracks.videoTrack) + } + if (this.localTracks.audioTrack) { + tracks.push(this.localTracks.audioTrack) + } + if (tracks.length) { + await this.client.publish(tracks) + } + } + + async destroy() { + this.localTracks?.audioTrack?.close() + this.localTracks?.videoTrack?.close() + if (this._joined) { + await this.client?.leave() + } + this._resetData() + } + + // ----------- public methods ------------ + + // -------------- private methods -------------- + private _listenRtcEvents() { + this.client.on("network-quality", (quality) => { + this.emit("networkQuality", quality) + }) + this.client.on("user-published", async (user, mediaType) => { + await this.client.subscribe(user, mediaType) + if (mediaType === "audio") { + this._playAudio(user.audioTrack) + } + this.emit("remoteUserChanged", { + userId: user.uid, + audioTrack: user.audioTrack, + videoTrack: user.videoTrack, + }) + }) + this.client.on("user-unpublished", async (user, mediaType) => { + await this.client.unsubscribe(user, mediaType) + this.emit("remoteUserChanged", { + userId: user.uid, + audioTrack: user.audioTrack, + videoTrack: user.videoTrack, + }) + }) + this.client.on("stream-message", (uid: UID, stream: any) => { + this._parseData(stream) + }) + } + + private _parseData(data: any): ITextItem | void { + let decoder = new TextDecoder('utf-8'); + let decodedMessage = decoder.decode(data); + const textstream = JSON.parse(decodedMessage); + + console.log("[test] textstream raw data", JSON.stringify(textstream)); + + const { stream_id, is_final, text, text_ts, data_type, message_id, part_number, total_parts } = textstream; + + if (total_parts > 0) { + // If message is split, handle it accordingly + this._handleSplitMessage(message_id, part_number, total_parts, stream_id, is_final, text, text_ts); + } else { + // If there is no message_id, treat it as a complete message + this._handleCompleteMessage(stream_id, is_final, text, text_ts); + } + } + + private messageCache: { [key: string]: { parts: string[], totalParts: number } } = {}; + + /** + * Handle complete messages (not split). + */ + private _handleCompleteMessage(stream_id: number, is_final: boolean, text: string, text_ts: number): void { + const textItem: ITextItem = { + uid: `${stream_id}`, + time: text_ts, + dataType: "transcribe", + text: text, + isFinal: is_final + }; + + if (text.trim().length > 0) { + this.emit("textChanged", textItem); + } + } + + /** + * Handle split messages, track parts, and reassemble once all parts are received. + */ + private _handleSplitMessage( + message_id: string, + part_number: number, + total_parts: number, + stream_id: number, + is_final: boolean, + text: string, + text_ts: number + ): void { + // Ensure the messageCache entry exists for this message_id + if (!this.messageCache[message_id]) { + this.messageCache[message_id] = { parts: [], totalParts: total_parts }; + } + + const cache = this.messageCache[message_id]; + + // Store the received part at the correct index (part_number starts from 1, so we use part_number - 1) + cache.parts[part_number - 1] = text; + + // Check if all parts have been received + const receivedPartsCount = cache.parts.filter(part => part !== undefined).length; + + if (receivedPartsCount === total_parts) { + // All parts have been received, reassemble the message + const fullText = cache.parts.join(''); + + // Now that the message is reassembled, handle it like a complete message + this._handleCompleteMessage(stream_id, is_final, fullText, text_ts); + + // Remove the cached message since it is now fully processed + delete this.messageCache[message_id]; + } + } + + + _playAudio(audioTrack: IMicrophoneAudioTrack | IRemoteAudioTrack | undefined) { + if (audioTrack && !audioTrack.isPlaying) { + audioTrack.play() + } + } + + + private _resetData() { + this.localTracks = {} + this._joined = false + } +} + + +export const rtcManager = new RtcManager() diff --git a/demo/src/manager/rtc/types.ts b/demo/src/manager/rtc/types.ts new file mode 100644 index 00000000..15e3f515 --- /dev/null +++ b/demo/src/manager/rtc/types.ts @@ -0,0 +1,25 @@ +import { + UID, + IAgoraRTCRemoteUser, + IAgoraRTCClient, + ICameraVideoTrack, + IMicrophoneAudioTrack, + NetworkQuality, +} from "agora-rtc-sdk-ng" +import { ITextItem } from "@/types" + +export interface IRtcUser extends IUserTracks { + userId: UID +} + +export interface RtcEvents { + remoteUserChanged: (user: IRtcUser) => void + localTracksChanged: (tracks: IUserTracks) => void + networkQuality: (quality: NetworkQuality) => void + textChanged: (text: ITextItem) => void +} + +export interface IUserTracks { + videoTrack?: ICameraVideoTrack + audioTrack?: IMicrophoneAudioTrack +} diff --git a/demo/src/manager/types.ts b/demo/src/manager/types.ts new file mode 100644 index 00000000..50e5b1c0 --- /dev/null +++ b/demo/src/manager/types.ts @@ -0,0 +1 @@ +export type EventHandler = (...data: T) => void diff --git a/demo/src/middleware.tsx b/demo/src/middleware.tsx new file mode 100644 index 00000000..724e0b4d --- /dev/null +++ b/demo/src/middleware.tsx @@ -0,0 +1,44 @@ +// middleware.js +import { NextRequest, NextResponse } from 'next/server'; + + +const { AGENT_SERVER_URL } = process.env; + +// Check if environment variables are available +if (!AGENT_SERVER_URL) { + throw "Environment variables AGENT_SERVER_URL are not available"; +} + +export function middleware(req: NextRequest) { + const { pathname } = req.nextUrl; + + if (pathname.startsWith('/api/agents/')) { + if (!pathname.startsWith('/api/agents/start')) { + + // Proxy all other agents API requests + const url = req.nextUrl.clone(); + url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/agents/', '/')}`; + + // console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + } + } else if (pathname.startsWith('/api/vector/')) { + + // Proxy all other documents requests + const url = req.nextUrl.clone(); + url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/vector/', '/vector/')}`; + + // console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + } else if (pathname.startsWith('/api/token/')) { + // Proxy all other documents requests + const url = req.nextUrl.clone(); + url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/token/', '/token/')}`; + + // console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + } else { + return NextResponse.next(); + } + +} \ No newline at end of file diff --git a/demo/src/platform/mobile/chat/chatItem/index.module.scss b/demo/src/platform/mobile/chat/chatItem/index.module.scss new file mode 100644 index 00000000..27057120 --- /dev/null +++ b/demo/src/platform/mobile/chat/chatItem/index.module.scss @@ -0,0 +1,86 @@ +.agentChatItem { + width: 100%; + display: flex; + justify-content: flex-start; + + .left { + flex: 0 0 auto; + display: flex; + width: 32px; + height: 32px; + padding: 10px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + border-radius: 200px; + background: var(--Grey-700, #475467); + + .userName { + color: var(---white, #FFF); + text-align: center; + font-size: 14px; + font-weight: 500; + line-height: 150%; + } + } + + .right { + margin-left: 12px; + + .userName { + font-size: 14px; + font-weight: 500; + line-height: 20px; + color: var(--theme-color, #667085) !important; + } + + + .agent { + color: var(--theme-color, #EAECF0) !important; + } + + } +} + +.userChatItem { + width: 100%; + display: flex; + flex-direction: column; + justify-content: flex-end; + align-items: flex-end; + + .userName { + text-align: right; + color: var(--Grey-600, #667085); + font-weight: 500; + line-height: 20px; + } + + + +} + + +.chatItem { + .text { + margin-top: 6px; + color: #FFF; + display: flex; + padding: 8px 14px; + flex-direction: column; + justify-content: left; + font-size: 14px; + font-weight: 400; + line-height: 21px; + white-space: pre-wrap; + border-radius: 0px 8px 8px 8px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} + +.chatItem+.chatItem { + margin-top: 14px; +} diff --git a/demo/src/platform/mobile/chat/chatItem/index.tsx b/demo/src/platform/mobile/chat/chatItem/index.tsx new file mode 100644 index 00000000..bde3350c --- /dev/null +++ b/demo/src/platform/mobile/chat/chatItem/index.tsx @@ -0,0 +1,50 @@ +import { IChatItem } from "@/types" +import styles from "./index.module.scss" + +interface ChatItemProps { + data: IChatItem +} + + +const AgentChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + + return
+ + Ag + + +
Agent
+
+ {text} +
+ +
+} + +const UserChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + return
+
You
+
{text}
+
+} + + +const ChatItem = (props: ChatItemProps) => { + const { data } = props + + + return ( + data.type === "agent" ? : + ); + + +} + + +export default ChatItem diff --git a/demo/src/platform/mobile/chat/index.module.scss b/demo/src/platform/mobile/chat/index.module.scss new file mode 100644 index 00000000..8ded1f2c --- /dev/null +++ b/demo/src/platform/mobile/chat/index.module.scss @@ -0,0 +1,78 @@ +.chat { + flex: 1 1 auto; + display: flex; + flex-direction: column; + align-items: flex-start; + align-self: stretch; + background: #181A1D; + overflow: hidden; + + .header { + display: flex; + flex-direction: column; + align-items: stretch; + row-gap: 10px; + border-bottom: 1px solid #272A2F; + width: 100%; + + + .text { + margin-left: 4px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + height: 40px; + line-height: 40px; + letter-spacing: 0.449px; + } + + .languageSelect { + width: 100%; + } + + + + + } + + .content { + margin-top: 16px; + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 10px; + align-self: stretch; + overflow-y: auto; + + + &::-webkit-scrollbar { + width: 6px + } + + &::-webkit-scrollbar-track { + background-color: transparent; + } + + &::-webkit-scrollbar-thumb { + background-color: #6B6B6B; + border-radius: 4px; + } + } + + +} + + +.dropdownRender { + display: flex; + justify-content: flex-end; + + + .btn { + color: var(--theme-color, #EAECF0); + + &:hover { + color: var(--theme-color, #EAECF0) !important; + } + } +} \ No newline at end of file diff --git a/demo/src/platform/mobile/chat/index.tsx b/demo/src/platform/mobile/chat/index.tsx new file mode 100644 index 00000000..bb071d4e --- /dev/null +++ b/demo/src/platform/mobile/chat/index.tsx @@ -0,0 +1,65 @@ +import { ReactElement, useEffect, useContext, useState } from "react" +import ChatItem from "./chatItem" +import { IChatItem } from "@/types" +import { useAppDispatch, useAutoScroll, LANGUAGE_OPTIONS, useAppSelector, GRAPH_OPTIONS, isRagGraph } from "@/common" +import { setGraphName, setLanguage } from "@/store/reducers/global" +import { Select, } from 'antd'; +import { MenuContext } from "../menu/context" +import PdfSelect from "@/components/pdfSelect" + +import styles from "./index.module.scss" + + +const Chat = () => { + const chatItems = useAppSelector(state => state.global.chatItems) + const language = useAppSelector(state => state.global.language) + const agentConnected = useAppSelector(state => state.global.agentConnected) + const graphName = useAppSelector(state => state.global.graphName) + const dispatch = useAppDispatch() + // genRandomChatList + // const [chatItems, setChatItems] = useState([]) + const context = useContext(MenuContext); + + if (!context) { + throw new Error("MenuContext is not found") + } + + const { scrollToBottom } = context; + + + useEffect(() => { + scrollToBottom() + }, [chatItems, scrollToBottom]) + + + + const onLanguageChange = (val: any) => { + dispatch(setLanguage(val)) + } + + const onGraphNameChange = (val: any) => { + dispatch(setGraphName(val)) + } + + + return
+
+ + + {isRagGraph(graphName) ? : null} +
+
+ {chatItems.map((item, index) => { + return + })} +
+
+} + + +export default Chat diff --git a/demo/src/platform/mobile/description/index.module.scss b/demo/src/platform/mobile/description/index.module.scss new file mode 100644 index 00000000..7305f5a7 --- /dev/null +++ b/demo/src/platform/mobile/description/index.module.scss @@ -0,0 +1,71 @@ +.description { + position: relative; + display: flex; + padding: 12px 16px; + height: 60px; + align-items: center; + gap: 12px; + align-self: stretch; + border-bottom: 1px solid #272A2F; + background: #181A1D; + box-sizing: border-box; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-style: normal; + font-weight: 600; + flex: 1 1 auto; + /* 21px */ + letter-spacing: 0.449px; + } + + .text { + margin-left: 12px; + flex: 1 1 auto; + color: var(--Grey-600, #667085); + font-size: 14px; + font-style: normal; + font-weight: 400; + } + + + .btnConnect { + width: 150px; + display: flex; + padding: 8px 14px; + justify-content: center; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 6px; + background: var(--theme-color, #0888FF); + box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05); + cursor: pointer; + user-select: none; + caret-color: transparent; + box-sizing: border-box; + + .btnText { + color: var(---White, #FFF); + font-size: 14px; + font-weight: 500; + line-height: 20px; + } + + .btnText.disconnect { + color: var(--Error-400-T, #E95C7B); + } + + .loading { + margin-left: 4px; + } + } + + + .btnConnect.disconnect { + background: #181A1D; + border: 1px solid var(--Error-400-T, #E95C7B); + } + +} diff --git a/demo/src/platform/mobile/description/index.tsx b/demo/src/platform/mobile/description/index.tsx new file mode 100644 index 00000000..7473d550 --- /dev/null +++ b/demo/src/platform/mobile/description/index.tsx @@ -0,0 +1,100 @@ +import { setAgentConnected } from "@/store/reducers/global" +import { + DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, + apiStartService, apiStopService +} from "@/common" +import { message } from "antd" +import { useEffect, useState } from "react" +import { LoadingOutlined, } from "@ant-design/icons" +import styles from "./index.module.scss" + +let intervalId: any + +const Description = () => { + const dispatch = useAppDispatch() + const agentConnected = useAppSelector(state => state.global.agentConnected) + const channel = useAppSelector(state => state.global.options.channel) + const userId = useAppSelector(state => state.global.options.userId) + const language = useAppSelector(state => state.global.language) + const voiceType = useAppSelector(state => state.global.voiceType) + const graphName = useAppSelector(state => state.global.graphName) + const [loading, setLoading] = useState(false) + + useEffect(() => { + if (channel) { + checkAgentConnected() + } + }, [channel]) + + + const checkAgentConnected = async () => { + const res: any = await apiPing(channel) + if (res?.code == 0) { + dispatch(setAgentConnected(true)) + } + } + + const onClickConnect = async () => { + if (loading) { + return + } + setLoading(true) + if (agentConnected) { + await apiStopService(channel) + dispatch(setAgentConnected(false)) + message.success("Agent disconnected") + stopPing() + } else { + const res = await apiStartService({ + channel, + userId, + graphName, + language, + voiceType + }) + const { code, msg } = res || {} + if (code != 0) { + if (code == "10001") { + message.error("The number of users experiencing the program simultaneously has exceeded the limit. Please try again later.") + } else { + message.error(`code:${code},msg:${msg}`) + } + setLoading(false) + throw new Error(msg) + } + dispatch(setAgentConnected(true)) + message.success("Agent connected") + startPing() + } + setLoading(false) + } + + const startPing = () => { + if (intervalId) { + stopPing() + } + intervalId = setInterval(() => { + apiPing(channel) + }, 3000) + } + + const stopPing = () => { + if (intervalId) { + clearInterval(intervalId) + intervalId = null + } + } + + return
+ Description + + + {!agentConnected ? "Connect" : "Disconnect"} + {loading ? : null} + + +
+} + + +export default Description diff --git a/demo/src/platform/mobile/entry/index.module.scss b/demo/src/platform/mobile/entry/index.module.scss new file mode 100644 index 00000000..41322c12 --- /dev/null +++ b/demo/src/platform/mobile/entry/index.module.scss @@ -0,0 +1,18 @@ +.entry { + position: relative; + height: 100%; + box-sizing: border-box; + + .content { + position: relative; + padding: 16px; + box-sizing: border-box; + + + .body { + margin-top: 16px; + display: flex; + gap: 24px; + } + } +} diff --git a/demo/src/platform/mobile/entry/index.tsx b/demo/src/platform/mobile/entry/index.tsx new file mode 100644 index 00000000..c5f51d5c --- /dev/null +++ b/demo/src/platform/mobile/entry/index.tsx @@ -0,0 +1,30 @@ +import Chat from "../chat" +import Description from "../description" +import Rtc from "../rtc" +import Header from "../header" +import Menu, { IMenuData } from "../menu" +import styles from "./index.module.scss" + + +const MenuData: IMenuData[] = [{ + name: "Agent", + component: , +}, { + name: "Chat", + component: , +}] + + +const MobileEntry = () => { + + return
+
+ +
+ +
+
+} + + +export default MobileEntry diff --git a/demo/src/platform/mobile/header/index.module.scss b/demo/src/platform/mobile/header/index.module.scss new file mode 100644 index 00000000..707e4215 --- /dev/null +++ b/demo/src/platform/mobile/header/index.module.scss @@ -0,0 +1,57 @@ +.header { + display: flex; + width: 100%; + height: 48px; + padding: 16px; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #24262A; + background: #1E2024; + box-shadow: 0px 12px 16px -4px rgba(8, 15, 52, 0.06), 0px 4px 6px -2px rgba(8, 15, 52, 0.03); + box-sizing: border-box; + z-index: 999; + + .logoWrapper { + display: flex; + align-items: center; + + .text { + margin-left: 8px; + color: var(---white, #FFF); + text-align: right; + font-family: Inter; + font-size: 16px; + font-weight: 500; + } + } + + .content { + padding-left: 12px; + display: flex; + align-items: center; + justify-content: flex-start; + height: 48px; + flex: 1 1 auto; + color: var(--Grey-300, #EAECF0); + font-size: 16px; + font-weight: 500; + line-height: 48px; + letter-spacing: 0.449px; + text-align: center; + + .text { + margin-left: 4px; + font-size: 12px; + } + } + + .links { + display: flex; + align-items: center; + gap: 8px; + + span { + display: flex; + } + } +} diff --git a/demo/src/platform/mobile/header/index.tsx b/demo/src/platform/mobile/header/index.tsx new file mode 100644 index 00000000..6e56a017 --- /dev/null +++ b/demo/src/platform/mobile/header/index.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useAppSelector, GITHUB_URL, useSmallScreen } from "@/common" +import Network from "./network" +import InfoPopover from "./infoPopover" +import StylePopover from "./stylePopover" +import { GithubIcon, LogoIcon, InfoIcon, ColorPickerIcon } from "@/components/icons" + +import styles from "./index.module.scss" + +const Header = () => { + const themeColor = useAppSelector(state => state.global.themeColor) + const options = useAppSelector(state => state.global.options) + const { channel } = options + + + const onClickGithub = () => { + if (typeof window !== "undefined") { + window.open(GITHUB_URL, "_blank") + } + } + + + + return
+ + + + + + + {channel} + + +
+ + + + + + + +
+
+} + + +export default Header diff --git a/demo/src/platform/mobile/header/infoPopover/index.module.scss b/demo/src/platform/mobile/header/infoPopover/index.module.scss new file mode 100644 index 00000000..cd3f72f8 --- /dev/null +++ b/demo/src/platform/mobile/header/infoPopover/index.module.scss @@ -0,0 +1,43 @@ +.info { + display: flex; + padding: 12px 16px; + flex-direction: column; + align-items: flex-start; + gap: 8px; + align-self: stretch; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .item { + width: 100%; + display: flex; + justify-content: space-between; + align-items: center; + + .title { + color: var(--Grey-600, #667085); + font-size: 14px; + font-weight: 400; + line-height: 150%; + } + + .content { + color: var(--theme-color, #FFF); + font-size: 14px; + font-weight: 400; + line-height: 150%; + } + } + + .slider { + height: 1px; + width: 100%; + background-color: #0D0F12; + } +} diff --git a/demo/src/platform/mobile/header/infoPopover/index.tsx b/demo/src/platform/mobile/header/infoPopover/index.tsx new file mode 100644 index 00000000..cd451418 --- /dev/null +++ b/demo/src/platform/mobile/header/infoPopover/index.tsx @@ -0,0 +1,57 @@ +import { useMemo } from "react" +import { useAppSelector } from "@/common" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface InfoPopoverProps { + children?: React.ReactNode +} + +const InfoPopover = (props: InfoPopoverProps) => { + const { children } = props + const options = useAppSelector(state => state.global.options) + const { channel, userId } = options + + const roomConnected = useAppSelector(state => state.global.roomConnected) + const agentConnected = useAppSelector(state => state.global.agentConnected) + + const roomConnectedText = useMemo(() => { + return roomConnected ? "TRUE" : "FALSE" + }, [roomConnected]) + + const agentConnectedText = useMemo(() => { + return agentConnected ? "TRUE" : "FALSE" + }, [agentConnected]) + + + + const content =
+
INFO
+
+ Room + {channel} +
+
+ Participant + {userId} +
+
+
STATUS
+
+
Room connected
+
{roomConnectedText}
+
+
+
Agent connected
+
{agentConnectedText}
+
+
+ + + return {children} + +} + +export default InfoPopover diff --git a/demo/src/platform/mobile/header/network/index.module.scss b/demo/src/platform/mobile/header/network/index.module.scss new file mode 100644 index 00000000..e69de29b diff --git a/demo/src/platform/mobile/header/network/index.tsx b/demo/src/platform/mobile/header/network/index.tsx new file mode 100644 index 00000000..92b4e33b --- /dev/null +++ b/demo/src/platform/mobile/header/network/index.tsx @@ -0,0 +1,37 @@ +"use client"; + +import React from "react"; +import { rtcManager } from "@/manager" +import { NetworkQuality } from "agora-rtc-sdk-ng" +import { useEffect, useState } from "react" +import { NetworkIcon } from "@/components/icons" + +interface NetworkProps { + style?: React.CSSProperties +} + +const NetWork = (props: NetworkProps) => { + const { style } = props + + const [networkQuality, setNetworkQuality] = useState() + + useEffect(() => { + rtcManager.on("networkQuality", onNetworkQuality) + + return () => { + rtcManager.off("networkQuality", onNetworkQuality) + } + }, []) + + const onNetworkQuality = (quality: NetworkQuality) => { + setNetworkQuality(quality) + } + + return ( + + + + ) +} + +export default NetWork diff --git a/demo/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss b/demo/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss new file mode 100644 index 00000000..405e7781 --- /dev/null +++ b/demo/src/platform/mobile/header/stylePopover/colorPicker/index.module.scss @@ -0,0 +1,24 @@ +.colorPicker { + height: 24px; + display: flex; + align-items: center; + + :global(.react-colorful) { + width: 220px; + height: 8px; + } + + :global(.react-colorful__saturation) { + display: none; + } + + :global(.react-colorful__hue) { + border-radius: 8px !important; + height: 8px; + } + + :global(.react-colorful__pointer) { + width: 24px; + height: 24px; + } +} diff --git a/demo/src/platform/mobile/header/stylePopover/colorPicker/index.tsx b/demo/src/platform/mobile/header/stylePopover/colorPicker/index.tsx new file mode 100644 index 00000000..28163d77 --- /dev/null +++ b/demo/src/platform/mobile/header/stylePopover/colorPicker/index.tsx @@ -0,0 +1,22 @@ +"use client" + +import { HexColorPicker } from "react-colorful"; +import { useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import styles from "./index.module.scss"; + +const ColorPicker = () => { + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + const onColorChange = (color: string) => { + console.log(color); + dispatch(setThemeColor(color)) + }; + + return
+ +
+}; + +export default ColorPicker; diff --git a/demo/src/platform/mobile/header/stylePopover/index.module.scss b/demo/src/platform/mobile/header/stylePopover/index.module.scss new file mode 100644 index 00000000..defdcc12 --- /dev/null +++ b/demo/src/platform/mobile/header/stylePopover/index.module.scss @@ -0,0 +1,51 @@ +.info { + padding: 12px 16px; + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 16px; + align-self: stretch; + + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .color { + font-size: 0; + white-space: nowrap; + + .item { + position: relative; + display: inline-block; + width: 28px; + height: 28px; + border-radius: 4px; + border: 2px solid transparent; + font-size: 0; + cursor: pointer; + + .inner { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + width: 18px; + height: 18px; + border-radius: 2px; + box-sizing: border-box; + } + } + + .item+.item { + margin-left: 12px; + } + + } + + +} diff --git a/demo/src/platform/mobile/header/stylePopover/index.tsx b/demo/src/platform/mobile/header/stylePopover/index.tsx new file mode 100644 index 00000000..f8508323 --- /dev/null +++ b/demo/src/platform/mobile/header/stylePopover/index.tsx @@ -0,0 +1,54 @@ +import { useMemo } from "react" +import { COLOR_LIST, useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import ColorPicker from "./colorPicker" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface StylePopoverProps { + children?: React.ReactNode +} + +const StylePopover = (props: StylePopoverProps) => { + const { children } = props + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + + const onClickColor = (index: number) => { + const target = COLOR_LIST[index] + if (target.active !== themeColor) { + dispatch(setThemeColor(target.active)) + } + } + + const content =
+
STYLE
+
+ { + COLOR_LIST.map((item, index) => { + return onClickColor(index)} + className={styles.item} + key={index}> + + + }) + } +
+ +
+ + + return {children} + +} + +export default StylePopover diff --git a/demo/src/platform/mobile/menu/context.ts b/demo/src/platform/mobile/menu/context.ts new file mode 100644 index 00000000..41c52911 --- /dev/null +++ b/demo/src/platform/mobile/menu/context.ts @@ -0,0 +1,9 @@ +import { createContext } from "react" + +export interface MenuContextType { + scrollToBottom: () => void; +} + +export const MenuContext = createContext({ + scrollToBottom: () => { } +}); diff --git a/demo/src/platform/mobile/menu/index.module.scss b/demo/src/platform/mobile/menu/index.module.scss new file mode 100644 index 00000000..58b1b3fe --- /dev/null +++ b/demo/src/platform/mobile/menu/index.module.scss @@ -0,0 +1,69 @@ +.menu { + width: 100%; + border: 1px solid #272A2F; + border-radius: 4px; + background: #0F0F11; + overflow: hidden; + box-sizing: border-box; + + .header { + height: 40px; + overflow: hidden; + border-bottom: 1px solid #272A2F; + box-sizing: border-box; + + .menuItem { + height: 40px; + padding: 0 16px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 40px; + letter-spacing: 0.449px; + display: inline-block; + color: #667085; + background: #181A1E; + cursor: pointer; + border-right: 1px solid #272A2F; + box-sizing: border-box; + overflow: hidden; + background: #0F0F11; + } + + .active { + color: #EAECF0; + background: #181A1D; + } + } + + + .content { + position: relative; + background: #181A1D; + // header 48px + // description 60px + // paddingTop 16px 16px + // menu header 40px + height: calc(100vh - 48px - 60px - 32px - 40px - 2px); + overflow: hidden; + box-sizing: border-box; + + .item { + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; + padding: 16px; + z-index: -1; + overflow: auto; + visibility: hidden; + box-sizing: border-box; + } + + .active { + z-index: 1; + visibility: visible; + } + } +} diff --git a/demo/src/platform/mobile/menu/index.tsx b/demo/src/platform/mobile/menu/index.tsx new file mode 100644 index 00000000..2e20de78 --- /dev/null +++ b/demo/src/platform/mobile/menu/index.tsx @@ -0,0 +1,76 @@ +"use client" + +import { ReactElement, useEffect, useState, useRef, useMemo, useCallback } from "react" +import { useAutoScroll } from "@/common" +import { MenuContext } from "./context" +import styles from "./index.module.scss" + +export interface IMenuData { + name: string, + component: ReactElement +} + +export interface IMenuContentComponentPros { + scrollToBottom: () => void +} + +interface MenuProps { + data: IMenuData[] +} + + +const Menu = (props: MenuProps) => { + const { data } = props + const [activeIndex, setActiveIndex] = useState(0) + const contentRefList = useRef<(HTMLDivElement | null)[]>([]) + + const onClickItem = (index: number) => { + setActiveIndex(index) + } + + useEffect(() => { + scrollToTop() + }, [activeIndex]) + + const scrollToBottom = useCallback(() => { + const current = contentRefList.current?.[activeIndex] + if (current) { + current.scrollTop = current.scrollHeight + } + }, [contentRefList, activeIndex]) + + const scrollToTop = useCallback(() => { + const current = contentRefList.current?.[activeIndex] + if (current) { + current.scrollTop = 0 + } + }, [contentRefList, activeIndex]) + + + return
+
+ {data.map((item, index) => { + return onClickItem(index)}>{item.name} + })} +
+
+ + {data.map((item, index) => { + return
{ + contentRefList.current[index] = el; + }} + className={`${styles.item} ${index == activeIndex ? styles.active : ''}`}> + {item.component} +
+ })} + +
+
+} + +export default Menu diff --git a/demo/src/platform/mobile/rtc/agent/index.module.scss b/demo/src/platform/mobile/rtc/agent/index.module.scss new file mode 100644 index 00000000..fa3ae2ec --- /dev/null +++ b/demo/src/platform/mobile/rtc/agent/index.module.scss @@ -0,0 +1,31 @@ +.agent { + position: relative; + display: flex; + height: 292px; + padding: 20px 16px; + flex-direction: column; + justify-content: flex-start; + align-items: center; + align-self: stretch; + background: linear-gradient(154deg, rgba(27, 66, 166, 0.16) 0%, rgba(27, 45, 140, 0.00) 18%), linear-gradient(153deg, rgba(23, 24, 28, 0.00) 53.75%, #11174E 100%), #0F0F11; + box-shadow: 0px 3.999px 48.988px 0px rgba(0, 7, 72, 0.12); + backdrop-filter: blur(7); + box-sizing: border-box; + + .text { + margin-top: 50px; + color: var(--theme-color, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .view { + margin-top: 32px; + display: flex; + align-items: center; + justify-content: center; + height: 56px; + } +} diff --git a/demo/src/platform/mobile/rtc/agent/index.tsx b/demo/src/platform/mobile/rtc/agent/index.tsx new file mode 100644 index 00000000..a7fd7944 --- /dev/null +++ b/demo/src/platform/mobile/rtc/agent/index.tsx @@ -0,0 +1,34 @@ +"use client" + +import { useAppSelector, useMultibandTrackVolume } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import styles from "./index.module.scss" + +interface AgentProps { + audioTrack?: IMicrophoneAudioTrack +} + +const Agent = (props: AgentProps) => { + const { audioTrack } = props + + const subscribedVolumes = useMultibandTrackVolume(audioTrack, 12); + + return
+
Agent
+
+ +
+
+ +} + + +export default Agent; diff --git a/demo/src/platform/mobile/rtc/audioVisualizer/index.module.scss b/demo/src/platform/mobile/rtc/audioVisualizer/index.module.scss new file mode 100644 index 00000000..1beae944 --- /dev/null +++ b/demo/src/platform/mobile/rtc/audioVisualizer/index.module.scss @@ -0,0 +1,17 @@ +.audioVisualizer { + display: flex; + justify-content: center; + align-items: center; + + + .item {} + + .agent { + background-color: var(--theme-color, #EAECF0); + box-shadow: 0 0 10px var(--theme-color, #EAECF0); + } + + .user { + background-color: var(--Grey-300, #EAECF0); + } +} diff --git a/demo/src/platform/mobile/rtc/audioVisualizer/index.tsx b/demo/src/platform/mobile/rtc/audioVisualizer/index.tsx new file mode 100644 index 00000000..bc21f554 --- /dev/null +++ b/demo/src/platform/mobile/rtc/audioVisualizer/index.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useState, useEffect } from "react" +import styles from "./index.module.scss" + +interface AudioVisualizerProps { + type: "agent" | "user"; + frequencies: Float32Array[]; + gap: number; + barWidth: number; + minBarHeight: number; + maxBarHeight: number + borderRadius: number; +} + + +const AudioVisualizer = (props: AudioVisualizerProps) => { + const { frequencies, gap, barWidth, minBarHeight, maxBarHeight, borderRadius, type } = props; + + const summedFrequencies = frequencies.map((bandFrequencies) => { + const sum = bandFrequencies.reduce((a, b) => a + b, 0) + if (sum <= 0) { + return 0 + } + return Math.sqrt(sum / bandFrequencies.length); + }); + + return
{ + summedFrequencies.map((frequency, index) => { + + const style = { + height: minBarHeight + frequency * (maxBarHeight - minBarHeight) + "px", + borderRadius: borderRadius + "px", + width: barWidth + "px", + transition: + "background-color 0.35s ease-out, transform 0.25s ease-out", + // transform: transform, + } + + return + }) + }
+} + + +export default AudioVisualizer; diff --git a/demo/src/platform/mobile/rtc/camSection/camSelect/index.module.scss b/demo/src/platform/mobile/rtc/camSection/camSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/demo/src/platform/mobile/rtc/camSection/camSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/demo/src/platform/mobile/rtc/camSection/camSelect/index.tsx b/demo/src/platform/mobile/rtc/camSection/camSelect/index.tsx new file mode 100644 index 00000000..33a5e003 --- /dev/null +++ b/demo/src/platform/mobile/rtc/camSection/camSelect/index.tsx @@ -0,0 +1,57 @@ +"use client" + +import AgoraRTC, { ICameraVideoTrack } from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" + +import styles from "./index.module.scss" + +interface CamSelectProps { + videoTrack?: ICameraVideoTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const CamSelect = (props: CamSelectProps) => { + const { videoTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (videoTrack) { + const label = videoTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getCameras().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [videoTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (videoTrack) { + await videoTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default CamSelect diff --git a/demo/src/platform/mobile/rtc/camSection/index.module.scss b/demo/src/platform/mobile/rtc/camSection/index.module.scss new file mode 100644 index 00000000..76f4ad1e --- /dev/null +++ b/demo/src/platform/mobile/rtc/camSection/index.module.scss @@ -0,0 +1,54 @@ +.camera { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .title { + margin-bottom: 10px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 500; + line-height: 150%; + letter-spacing: 0.449px; + } + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + .select { + flex: 0 0 auto; + width: 200px; + } + } + + .view { + position: relative; + margin-top: 12px; + min-height: 210px; + height: 210px; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/demo/src/platform/mobile/rtc/camSection/index.tsx b/demo/src/platform/mobile/rtc/camSection/index.tsx new file mode 100644 index 00000000..2bd0e8db --- /dev/null +++ b/demo/src/platform/mobile/rtc/camSection/index.tsx @@ -0,0 +1,42 @@ +"use client" + +import CamSelect from "./camSelect" +import { CamIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { ICameraVideoTrack } from 'agora-rtc-sdk-ng'; +import { LocalStreamPlayer } from "../streamPlayer" +import { useState, useEffect, useMemo } from 'react'; +import { useSmallScreen } from "@/common" + +interface CamSectionProps { + videoTrack?: ICameraVideoTrack +} + +const CamSection = (props: CamSectionProps) => { + const { videoTrack } = props + const [videoMute, setVideoMute] = useState(false) + + useEffect(() => { + videoTrack?.setMuted(videoMute) + }, [videoTrack, videoMute]) + + const onClickMute = () => { + setVideoMute(!videoMute) + } + + return
+
CAMERA
+
+ + + + +
+
+ +
+
+} + + +export default CamSection; diff --git a/demo/src/platform/mobile/rtc/index.module.scss b/demo/src/platform/mobile/rtc/index.module.scss new file mode 100644 index 00000000..ff7b7958 --- /dev/null +++ b/demo/src/platform/mobile/rtc/index.module.scss @@ -0,0 +1,55 @@ +.rtc { + flex: 0 0 420px; + display: flex; + flex-direction: column; + align-items: flex-start; + flex-shrink: 0; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + box-sizing: border-box; + + .header { + display: flex; + height: 40px; + padding: 0px 16px; + align-items: center; + align-self: stretch; + border-bottom: 1px solid #272A2F; + + .text { + flex: 1 1 auto; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + color: var(--Grey-300, #EAECF0); + } + + .voiceSelect { + flex: 0 0 120px; + } + } + + .you { + display: flex; + padding: 24px 16px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 24px; + align-self: stretch; + border-top: 1px solid #272A2F; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + text-align: center; + } + + + } +} diff --git a/demo/src/platform/mobile/rtc/index.tsx b/demo/src/platform/mobile/rtc/index.tsx new file mode 100644 index 00000000..bc15c070 --- /dev/null +++ b/demo/src/platform/mobile/rtc/index.tsx @@ -0,0 +1,128 @@ +"use client" + +import { ICameraVideoTrack, IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" +import { useAppSelector, useAppDispatch, VOICE_OPTIONS } from "@/common" +import { ITextItem } from "@/types" +import { rtcManager, IUserTracks, IRtcUser } from "@/manager" +import { setRoomConnected, addChatItem, setVoiceType } from "@/store/reducers/global" +import MicSection from "./micSection" +import CamSection from "./camSection" +import Agent from "./agent" +import styles from "./index.module.scss" +import { useRef, useEffect, useState, Fragment } from "react" +import { VoiceIcon } from "@/components/icons" +import CustomSelect from "@/components/customSelect" + +let hasInit = false + +const Rtc = () => { + const dispatch = useAppDispatch() + const options = useAppSelector(state => state.global.options) + const voiceType = useAppSelector(state => state.global.voiceType) + const agentConnected = useAppSelector(state => state.global.agentConnected) + const { userId, channel } = options + const [videoTrack, setVideoTrack] = useState() + const [audioTrack, setAudioTrack] = useState() + const [remoteuser, setRemoteUser] = useState() + + useEffect(() => { + if (!options.channel) { + return + } + if (hasInit) { + return + } + + init() + + return () => { + if (hasInit) { + destory() + } + } + }, [options.channel]) + + + const init = async () => { + console.log("[test] init") + rtcManager.on("localTracksChanged", onLocalTracksChanged) + rtcManager.on("textChanged", onTextChanged) + rtcManager.on("remoteUserChanged", onRemoteUserChanged) + await rtcManager.createTracks() + await rtcManager.join({ + channel, + userId + }) + await rtcManager.publish() + dispatch(setRoomConnected(true)) + hasInit = true + } + + const destory = async () => { + console.log("[test] destory") + rtcManager.off("textChanged", onTextChanged) + rtcManager.off("localTracksChanged", onLocalTracksChanged) + rtcManager.off("remoteUserChanged", onRemoteUserChanged) + await rtcManager.destroy() + dispatch(setRoomConnected(false)) + hasInit = false + } + + const onRemoteUserChanged = (user: IRtcUser) => { + console.log("[test] onRemoteUserChanged", user) + setRemoteUser(user) + } + + const onLocalTracksChanged = (tracks: IUserTracks) => { + console.log("[test] onLocalTracksChanged", tracks) + const { videoTrack, audioTrack } = tracks + if (videoTrack) { + setVideoTrack(videoTrack) + } + if (audioTrack) { + setAudioTrack(audioTrack) + } + } + + const onTextChanged = (text: ITextItem) => { + if (text.dataType == "transcribe") { + const isAgent = Number(text.uid) != Number(userId) + dispatch(addChatItem({ + userId: text.uid, + text: text.text, + type: isAgent ? "agent" : "user", + isFinal: text.isFinal, + time: text.time + })) + } + } + + const onVoiceChange = (value: any) => { + dispatch(setVoiceType(value)) + } + + + return
+
+ Audio & Video + } + options={VOICE_OPTIONS} onChange={onVoiceChange}> +
+ {/* agent */} + + {/* you */} +
+
You
+ {/* microphone */} + + {/* camera */} + +
+
+} + + +export default Rtc; diff --git a/demo/src/platform/mobile/rtc/micSection/index.module.scss b/demo/src/platform/mobile/rtc/micSection/index.module.scss new file mode 100644 index 00000000..60cc6fe1 --- /dev/null +++ b/demo/src/platform/mobile/rtc/micSection/index.module.scss @@ -0,0 +1,58 @@ +.microphone { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .title { + margin-bottom: 10px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 500; + line-height: 150%; + letter-spacing: 0.449px; + } + + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + + } + + .view { + margin-top: 12px; + display: flex; + height: 120px; + padding: 24px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + align-self: stretch; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/demo/src/platform/mobile/rtc/micSection/index.tsx b/demo/src/platform/mobile/rtc/micSection/index.tsx new file mode 100644 index 00000000..3c739159 --- /dev/null +++ b/demo/src/platform/mobile/rtc/micSection/index.tsx @@ -0,0 +1,70 @@ +"use client" + +import { useEffect, useMemo, useState } from "react" +import { useMultibandTrackVolume, useSmallScreen } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { MicIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import MicSelect from "./micSelect"; + +interface MicSectionProps { + audioTrack?: IMicrophoneAudioTrack +} + +const MicSection = (props: MicSectionProps) => { + const { audioTrack } = props + const [audioMute, setAudioMute] = useState(false) + const [mediaStreamTrack, setMediaStreamTrack] = useState() + + + + useEffect(() => { + audioTrack?.on("track-updated", onAudioTrackupdated) + if (audioTrack) { + setMediaStreamTrack(audioTrack.getMediaStreamTrack()) + } + + return () => { + audioTrack?.off("track-updated", onAudioTrackupdated) + } + }, [audioTrack]) + + useEffect(() => { + audioTrack?.setMuted(audioMute) + }, [audioTrack, audioMute]) + + const subscribedVolumes = useMultibandTrackVolume(mediaStreamTrack, 20); + + const onAudioTrackupdated = (track: MediaStreamTrack) => { + console.log("[test] audio track updated", track) + setMediaStreamTrack(track) + } + + const onClickMute = () => { + setAudioMute(!audioMute) + } + + return
+
MICROPHONE
+
+ + + + +
+
+ +
+
+} + + +export default MicSection; diff --git a/demo/src/platform/mobile/rtc/micSection/micSelect/index.module.scss b/demo/src/platform/mobile/rtc/micSection/micSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/demo/src/platform/mobile/rtc/micSection/micSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/demo/src/platform/mobile/rtc/micSection/micSelect/index.tsx b/demo/src/platform/mobile/rtc/micSection/micSelect/index.tsx new file mode 100644 index 00000000..efc842b5 --- /dev/null +++ b/demo/src/platform/mobile/rtc/micSection/micSelect/index.tsx @@ -0,0 +1,58 @@ +"use client" + +import AgoraRTC from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" +import { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" + +import styles from "./index.module.scss" + +interface MicSelectProps { + audioTrack?: IMicrophoneAudioTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const MicSelect = (props: MicSelectProps) => { + const { audioTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (audioTrack) { + const label = audioTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getMicrophones().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [audioTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (audioTrack) { + await audioTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default MicSelect diff --git a/demo/src/platform/mobile/rtc/streamPlayer/index.module.scss b/demo/src/platform/mobile/rtc/streamPlayer/index.module.scss new file mode 100644 index 00000000..b1c57c10 --- /dev/null +++ b/demo/src/platform/mobile/rtc/streamPlayer/index.module.scss @@ -0,0 +1,6 @@ +.streamPlayer { + position: relative; + width: 100%; + height: 100%; + overflow: hidden; +} diff --git a/demo/src/platform/mobile/rtc/streamPlayer/index.tsx b/demo/src/platform/mobile/rtc/streamPlayer/index.tsx new file mode 100644 index 00000000..ba78e377 --- /dev/null +++ b/demo/src/platform/mobile/rtc/streamPlayer/index.tsx @@ -0,0 +1 @@ +export * from "./localStreamPlayer" diff --git a/demo/src/platform/mobile/rtc/streamPlayer/localStreamPlayer.tsx b/demo/src/platform/mobile/rtc/streamPlayer/localStreamPlayer.tsx new file mode 100644 index 00000000..e3e7f06a --- /dev/null +++ b/demo/src/platform/mobile/rtc/streamPlayer/localStreamPlayer.tsx @@ -0,0 +1,46 @@ +"use client" + +import { + ICameraVideoTrack, + IMicrophoneAudioTrack, + IRemoteAudioTrack, + IRemoteVideoTrack, + VideoPlayerConfig, +} from "agora-rtc-sdk-ng" +import { useRef, useState, useLayoutEffect, forwardRef, useEffect, useMemo } from "react" + +import styles from "./index.module.scss" + +interface StreamPlayerProps { + videoTrack?: ICameraVideoTrack + audioTrack?: IMicrophoneAudioTrack + style?: React.CSSProperties + fit?: "cover" | "contain" | "fill" + onClick?: () => void + mute?: boolean +} + +export const LocalStreamPlayer = forwardRef((props: StreamPlayerProps, ref) => { + const { videoTrack, audioTrack, mute = false, style = {}, fit = "cover", onClick = () => { } } = props + const vidDiv = useRef(null) + + useLayoutEffect(() => { + const config = { fit } as VideoPlayerConfig + if (mute) { + videoTrack?.stop() + } else { + if (!videoTrack?.isPlaying) { + videoTrack?.play(vidDiv.current!, config) + } + } + + return () => { + videoTrack?.stop() + } + }, [videoTrack, fit, mute]) + + // local audio track need not to be played + // useLayoutEffect(() => {}, [audioTrack, localAudioMute]) + + return
+}) diff --git a/demo/src/platform/pc/chat/chatItem/index.module.scss b/demo/src/platform/pc/chat/chatItem/index.module.scss new file mode 100644 index 00000000..f28ef7ee --- /dev/null +++ b/demo/src/platform/pc/chat/chatItem/index.module.scss @@ -0,0 +1,90 @@ +.agentChatItem { + width: 100%; + display: flex; + justify-content: flex-start; + + .left { + flex: 0 0 auto; + display: flex; + width: 32px; + height: 32px; + padding: 10px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + border-radius: 200px; + background: var(--Grey-700, #475467); + + .userName { + color: var(---white, #FFF); + text-align: center; + font-size: 14px; + font-weight: 500; + line-height: 150%; + } + } + + .right { + margin-left: 12px; + flex: 1 1 auto; + + .userName { + font-size: 14px; + font-weight: 500; + line-height: 20px; + color: var(--theme-color, #667085) !important; + } + + + .agent { + color: var(--theme-color, #EAECF0) !important; + } + + } +} + +.userChatItem { + width: 100%; + display: flex; + flex-direction: column; + justify-content: flex-end; + align-items: flex-end; + + .userName { + text-align: right; + color: var(--Grey-600, #667085); + font-weight: 500; + line-height: 20px; + } + + + +} + + +.chatItem { + .text { + max-width: 80%; + width: fit-content; + margin-top: 6px; + color: #FFF; + display: flex; + padding: 8px 14px; + flex-direction: column; + justify-content: center; + align-items: flex-start; + font-size: 14px; + font-weight: 400; + line-height: 21px; + white-space: pre-wrap; + border-radius: 0px 8px 8px 8px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} + +.chatItem+.chatItem { + margin-top: 14px; +} diff --git a/demo/src/platform/pc/chat/chatItem/index.tsx b/demo/src/platform/pc/chat/chatItem/index.tsx new file mode 100644 index 00000000..6364aaea --- /dev/null +++ b/demo/src/platform/pc/chat/chatItem/index.tsx @@ -0,0 +1,51 @@ +import { IChatItem } from "@/types" +import styles from "./index.module.scss" +import { usePrevious } from "@/common" +import { use, useEffect, useMemo, useState } from "react" + +interface ChatItemProps { + data: IChatItem +} + + +const AgentChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + return
+ + Ag + + +
Agent
+
+ {text} +
+ +
+} + +const UserChatItem = (props: ChatItemProps) => { + const { data } = props + const { text } = data + + return
+
You
+
{text}
+
+} + + +const ChatItem = (props: ChatItemProps) => { + const { data } = props + + + return ( + data.type === "agent" ? : + ); + + +} + + +export default ChatItem diff --git a/demo/src/platform/pc/chat/index.module.scss b/demo/src/platform/pc/chat/index.module.scss new file mode 100644 index 00000000..39c4956d --- /dev/null +++ b/demo/src/platform/pc/chat/index.module.scss @@ -0,0 +1,79 @@ +.chat { + flex: 1 1 auto; + min-width: 500px; + display: flex; + flex-direction: column; + align-items: flex-start; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + overflow: hidden; + + .header { + display: flex; + height: 42px; + padding: 0px 16px; + align-items: center; + align-self: stretch; + border-bottom: 1px solid #272A2F; + + .left { + flex: 1 1 auto; + display: flex; + align-items: center; + gap: 5px; + + .text { + margin-left: 4px; + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + height: 40px; + line-height: 40px; + letter-spacing: 0.449px; + } + + .languageSelect { + width: 100px; + } + } + + + .right { + display: flex; + align-items: center; + gap: 10px; + flex: 0 0 230px; + justify-content: right; + } + + } + + .content { + display: flex; + padding: 12px 24px; + flex-direction: column; + align-items: flex-start; + gap: 10px; + flex: 1 0 500px; + align-self: stretch; + overflow-y: auto; + + + &::-webkit-scrollbar { + width: 6px + } + + &::-webkit-scrollbar-track { + background-color: transparent; + } + + &::-webkit-scrollbar-thumb { + background-color: #6B6B6B; + border-radius: 4px; + } + } + + +} diff --git a/demo/src/platform/pc/chat/index.tsx b/demo/src/platform/pc/chat/index.tsx new file mode 100644 index 00000000..64dea171 --- /dev/null +++ b/demo/src/platform/pc/chat/index.tsx @@ -0,0 +1,66 @@ +"use client" + +import { ReactElement, useEffect, useRef, useState } from "react" +import ChatItem from "./chatItem" +import { + genRandomChatList, useAppDispatch, useAutoScroll, + LANGUAGE_OPTIONS, useAppSelector, + GRAPH_OPTIONS, + isRagGraph, +} from "@/common" +import { setGraphName, setLanguage } from "@/store/reducers/global" +import { Select, } from 'antd'; +import PdfSelect from "@/components/pdfSelect" + +import styles from "./index.module.scss" + + + + +const Chat = () => { + const dispatch = useAppDispatch() + const chatItems = useAppSelector(state => state.global.chatItems) + const language = useAppSelector(state => state.global.language) + const graphName = useAppSelector(state => state.global.graphName) + const agentConnected = useAppSelector(state => state.global.agentConnected) + + // const chatItems = genRandomChatList(10) + const chatRef = useRef(null) + + + useAutoScroll(chatRef) + + + const onLanguageChange = (val: any) => { + dispatch(setLanguage(val)) + } + + const onGraphNameChange = (val: any) => { + dispatch(setGraphName(val)) + } + + + return
+
+ + + + + + {isRagGraph(graphName) ? : null} + +
+
+ {chatItems.map((item, index) => { + return + })} +
+
+} + + +export default Chat diff --git a/demo/src/platform/pc/description/index.module.scss b/demo/src/platform/pc/description/index.module.scss new file mode 100644 index 00000000..50b29301 --- /dev/null +++ b/demo/src/platform/pc/description/index.module.scss @@ -0,0 +1,73 @@ +.description { + position: relative; + display: flex; + padding: 12px 16px; + align-items: center; + gap: 12px; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-style: normal; + font-weight: 600; + /* 21px */ + letter-spacing: 0.449px; + } + + .text { + margin-left: 12px; + flex: 1 1 auto; + color: var(--Grey-600, #667085); + font-size: 14px; + font-style: normal; + font-weight: 400; + } + + + .btnConnect { + width: 150px; + display: flex; + padding: 8px 14px; + justify-content: center; + align-items: center; + gap: 8px; + align-self: stretch; + border-radius: 6px; + background: var(--theme-color, #0888FF); + border: 1px solid var(--theme-color, #0888FF); + box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05); + cursor: pointer; + user-select: none; + caret-color: transparent; + box-sizing: border-box; + + .btnText { + width: 100px; + text-align: center; + color: var(---White, #FFF); + font-size: 14px; + font-weight: 500; + line-height: 20px; + } + + .btnText.disconnect { + color: var(--Error-400-T, #E95C7B); + } + + + .loading { + margin-left: 4px; + } + } + + + .btnConnect.disconnect { + background: #181A1D; + border: 1px solid var(--Error-400-T, #E95C7B); + } + +} diff --git a/demo/src/platform/pc/description/index.tsx b/demo/src/platform/pc/description/index.tsx new file mode 100644 index 00000000..a9a055cd --- /dev/null +++ b/demo/src/platform/pc/description/index.tsx @@ -0,0 +1,101 @@ +import { setAgentConnected } from "@/store/reducers/global" +import { + DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, + apiStartService, apiStopService +} from "@/common" +import { Select, Button, message, Upload } from "antd" +import { useEffect, useState, MouseEventHandler } from "react" +import { LoadingOutlined, UploadOutlined } from "@ant-design/icons" +import styles from "./index.module.scss" + +let intervalId: any + +const Description = () => { + const dispatch = useAppDispatch() + const agentConnected = useAppSelector(state => state.global.agentConnected) + const channel = useAppSelector(state => state.global.options.channel) + const userId = useAppSelector(state => state.global.options.userId) + const language = useAppSelector(state => state.global.language) + const voiceType = useAppSelector(state => state.global.voiceType) + const graphName = useAppSelector(state => state.global.graphName) + const [loading, setLoading] = useState(false) + + useEffect(() => { + if (channel) { + checkAgentConnected() + } + }, [channel]) + + + const checkAgentConnected = async () => { + const res: any = await apiPing(channel) + if (res?.code == 0) { + dispatch(setAgentConnected(true)) + } + } + + const onClickConnect = async () => { + if (loading) { + return + } + setLoading(true) + if (agentConnected) { + await apiStopService(channel) + dispatch(setAgentConnected(false)) + message.success("Agent disconnected") + stopPing() + } else { + const res = await apiStartService({ + channel, + userId, + graphName, + language, + voiceType + }) + const { code, msg } = res || {} + if (code != 0) { + if (code == "10001") { + message.error("The number of users experiencing the program simultaneously has exceeded the limit. Please try again later.") + } else { + message.error(`code:${code},msg:${msg}`) + } + setLoading(false) + throw new Error(msg) + } + dispatch(setAgentConnected(true)) + message.success("Agent connected") + startPing() + } + setLoading(false) + } + + const startPing = () => { + if (intervalId) { + stopPing() + } + intervalId = setInterval(() => { + apiPing(channel) + }, 3000) + } + + const stopPing = () => { + if (intervalId) { + clearInterval(intervalId) + intervalId = null + } + } + + return
+ Description + Astra is a multimodal agent powered by TEN + + + {!agentConnected ? "Connect" : "Disconnect"} + {loading ? : null} + + +
+} + + +export default Description diff --git a/demo/src/platform/pc/entry/index.module.scss b/demo/src/platform/pc/entry/index.module.scss new file mode 100644 index 00000000..f138183f --- /dev/null +++ b/demo/src/platform/pc/entry/index.module.scss @@ -0,0 +1,17 @@ +.entry { + position: relative; + height: 100%; + box-sizing: border-box; + + .content { + position: relative; + padding: 16px; + box-sizing: border-box; + + .body { + margin-top: 16px; + display: flex; + gap: 24px; + } + } +} diff --git a/demo/src/platform/pc/entry/index.tsx b/demo/src/platform/pc/entry/index.tsx new file mode 100644 index 00000000..e7acd7f1 --- /dev/null +++ b/demo/src/platform/pc/entry/index.tsx @@ -0,0 +1,22 @@ +import Chat from "../chat" +import Description from "../description" +import Rtc from "../rtc" +import Header from "../header" + +import styles from "./index.module.scss" + +const PCEntry = () => { + return
+
+
+ +
+ + +
+
+
+} + + +export default PCEntry diff --git a/demo/src/platform/pc/header/index.module.scss b/demo/src/platform/pc/header/index.module.scss new file mode 100644 index 00000000..31138cc7 --- /dev/null +++ b/demo/src/platform/pc/header/index.module.scss @@ -0,0 +1,58 @@ +.header { + display: flex; + width: 100%; + height: 48px; + padding: 24px; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #24262A; + background: #1E2024; + box-shadow: 0px 12px 16px -4px rgba(8, 15, 52, 0.06), 0px 4px 6px -2px rgba(8, 15, 52, 0.03); + box-sizing: border-box; + z-index: 999; + + .logoWrapper { + display: flex; + align-items: center; + + .text { + margin-left: 8px; + color: var(---white, #FFF); + text-align: right; + font-family: Inter; + font-size: 16px; + font-weight: 500; + } + } + + .content { + display: flex; + align-items: center; + justify-content: center; + height: 48px; + flex: 1 1 auto; + color: var(--Grey-300, #EAECF0); + font-size: 16px; + font-weight: 500; + line-height: 48px; + letter-spacing: 0.449px; + text-align: center; + + .text { + margin-left: 4px; + } + } + + .links { + display: flex; + align-items: center; + gap: 8px; + + span { + display: flex; + } + } + .githubWrapper { + cursor: pointer; + } +} diff --git a/demo/src/platform/pc/header/index.tsx b/demo/src/platform/pc/header/index.tsx new file mode 100644 index 00000000..a55b3f04 --- /dev/null +++ b/demo/src/platform/pc/header/index.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useAppSelector, GITHUB_URL, useSmallScreen } from "@/common" +import Network from "./network" +import InfoPopover from "./infoPopover" +import StylePopover from "./stylePopover" +import { GithubIcon, LogoIcon, InfoIcon, ColorPickerIcon } from "@/components/icons" + +import styles from "./index.module.scss" + +const Header = () => { + const themeColor = useAppSelector(state => state.global.themeColor) + const options = useAppSelector(state => state.global.options) + const { channel } = options + + + const onClickGithub = () => { + if (typeof window !== "undefined") { + window.open(GITHUB_URL, "_blank") + } + } + + + + return
+ + + + + + + Channel Name: {channel} + + +
+ + + + + + + +
+
+} + + +export default Header diff --git a/demo/src/platform/pc/header/infoPopover/index.module.scss b/demo/src/platform/pc/header/infoPopover/index.module.scss new file mode 100644 index 00000000..cd3f72f8 --- /dev/null +++ b/demo/src/platform/pc/header/infoPopover/index.module.scss @@ -0,0 +1,43 @@ +.info { + display: flex; + padding: 12px 16px; + flex-direction: column; + align-items: flex-start; + gap: 8px; + align-self: stretch; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .item { + width: 100%; + display: flex; + justify-content: space-between; + align-items: center; + + .title { + color: var(--Grey-600, #667085); + font-size: 14px; + font-weight: 400; + line-height: 150%; + } + + .content { + color: var(--theme-color, #FFF); + font-size: 14px; + font-weight: 400; + line-height: 150%; + } + } + + .slider { + height: 1px; + width: 100%; + background-color: #0D0F12; + } +} diff --git a/demo/src/platform/pc/header/infoPopover/index.tsx b/demo/src/platform/pc/header/infoPopover/index.tsx new file mode 100644 index 00000000..cd451418 --- /dev/null +++ b/demo/src/platform/pc/header/infoPopover/index.tsx @@ -0,0 +1,57 @@ +import { useMemo } from "react" +import { useAppSelector } from "@/common" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface InfoPopoverProps { + children?: React.ReactNode +} + +const InfoPopover = (props: InfoPopoverProps) => { + const { children } = props + const options = useAppSelector(state => state.global.options) + const { channel, userId } = options + + const roomConnected = useAppSelector(state => state.global.roomConnected) + const agentConnected = useAppSelector(state => state.global.agentConnected) + + const roomConnectedText = useMemo(() => { + return roomConnected ? "TRUE" : "FALSE" + }, [roomConnected]) + + const agentConnectedText = useMemo(() => { + return agentConnected ? "TRUE" : "FALSE" + }, [agentConnected]) + + + + const content =
+
INFO
+
+ Room + {channel} +
+
+ Participant + {userId} +
+
+
STATUS
+
+
Room connected
+
{roomConnectedText}
+
+
+
Agent connected
+
{agentConnectedText}
+
+
+ + + return {children} + +} + +export default InfoPopover diff --git a/demo/src/platform/pc/header/network/index.module.scss b/demo/src/platform/pc/header/network/index.module.scss new file mode 100644 index 00000000..e69de29b diff --git a/demo/src/platform/pc/header/network/index.tsx b/demo/src/platform/pc/header/network/index.tsx new file mode 100644 index 00000000..92b4e33b --- /dev/null +++ b/demo/src/platform/pc/header/network/index.tsx @@ -0,0 +1,37 @@ +"use client"; + +import React from "react"; +import { rtcManager } from "@/manager" +import { NetworkQuality } from "agora-rtc-sdk-ng" +import { useEffect, useState } from "react" +import { NetworkIcon } from "@/components/icons" + +interface NetworkProps { + style?: React.CSSProperties +} + +const NetWork = (props: NetworkProps) => { + const { style } = props + + const [networkQuality, setNetworkQuality] = useState() + + useEffect(() => { + rtcManager.on("networkQuality", onNetworkQuality) + + return () => { + rtcManager.off("networkQuality", onNetworkQuality) + } + }, []) + + const onNetworkQuality = (quality: NetworkQuality) => { + setNetworkQuality(quality) + } + + return ( + + + + ) +} + +export default NetWork diff --git a/demo/src/platform/pc/header/stylePopover/colorPicker/index.module.scss b/demo/src/platform/pc/header/stylePopover/colorPicker/index.module.scss new file mode 100644 index 00000000..405e7781 --- /dev/null +++ b/demo/src/platform/pc/header/stylePopover/colorPicker/index.module.scss @@ -0,0 +1,24 @@ +.colorPicker { + height: 24px; + display: flex; + align-items: center; + + :global(.react-colorful) { + width: 220px; + height: 8px; + } + + :global(.react-colorful__saturation) { + display: none; + } + + :global(.react-colorful__hue) { + border-radius: 8px !important; + height: 8px; + } + + :global(.react-colorful__pointer) { + width: 24px; + height: 24px; + } +} diff --git a/demo/src/platform/pc/header/stylePopover/colorPicker/index.tsx b/demo/src/platform/pc/header/stylePopover/colorPicker/index.tsx new file mode 100644 index 00000000..28163d77 --- /dev/null +++ b/demo/src/platform/pc/header/stylePopover/colorPicker/index.tsx @@ -0,0 +1,22 @@ +"use client" + +import { HexColorPicker } from "react-colorful"; +import { useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import styles from "./index.module.scss"; + +const ColorPicker = () => { + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + const onColorChange = (color: string) => { + console.log(color); + dispatch(setThemeColor(color)) + }; + + return
+ +
+}; + +export default ColorPicker; diff --git a/demo/src/platform/pc/header/stylePopover/index.module.scss b/demo/src/platform/pc/header/stylePopover/index.module.scss new file mode 100644 index 00000000..98c7f182 --- /dev/null +++ b/demo/src/platform/pc/header/stylePopover/index.module.scss @@ -0,0 +1,51 @@ +.info { + display: flex; + padding: 12px 16px; + flex-direction: column; + align-items: flex-start; + gap: 16px; + align-self: stretch; + + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 14px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .color { + font-size: 0; + white-space: nowrap; + + .item { + position: relative; + display: inline-block; + width: 28px; + height: 28px; + border-radius: 4px; + border: 2px solid transparent; + font-size: 0; + cursor: pointer; + + .inner { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + width: 18px; + height: 18px; + border-radius: 2px; + box-sizing: border-box; + } + } + + .item+.item { + margin-left: 12px; + } + + } + + +} diff --git a/demo/src/platform/pc/header/stylePopover/index.tsx b/demo/src/platform/pc/header/stylePopover/index.tsx new file mode 100644 index 00000000..f8508323 --- /dev/null +++ b/demo/src/platform/pc/header/stylePopover/index.tsx @@ -0,0 +1,54 @@ +import { useMemo } from "react" +import { COLOR_LIST, useAppSelector, useAppDispatch } from "@/common" +import { setThemeColor } from "@/store/reducers/global" +import ColorPicker from "./colorPicker" +import { Popover } from 'antd'; + + +import styles from "./index.module.scss" + +interface StylePopoverProps { + children?: React.ReactNode +} + +const StylePopover = (props: StylePopoverProps) => { + const { children } = props + const dispatch = useAppDispatch() + const themeColor = useAppSelector(state => state.global.themeColor) + + + const onClickColor = (index: number) => { + const target = COLOR_LIST[index] + if (target.active !== themeColor) { + dispatch(setThemeColor(target.active)) + } + } + + const content =
+
STYLE
+
+ { + COLOR_LIST.map((item, index) => { + return onClickColor(index)} + className={styles.item} + key={index}> + + + }) + } +
+ +
+ + + return {children} + +} + +export default StylePopover diff --git a/demo/src/platform/pc/rtc/agent/index.module.scss b/demo/src/platform/pc/rtc/agent/index.module.scss new file mode 100644 index 00000000..fa3ae2ec --- /dev/null +++ b/demo/src/platform/pc/rtc/agent/index.module.scss @@ -0,0 +1,31 @@ +.agent { + position: relative; + display: flex; + height: 292px; + padding: 20px 16px; + flex-direction: column; + justify-content: flex-start; + align-items: center; + align-self: stretch; + background: linear-gradient(154deg, rgba(27, 66, 166, 0.16) 0%, rgba(27, 45, 140, 0.00) 18%), linear-gradient(153deg, rgba(23, 24, 28, 0.00) 53.75%, #11174E 100%), #0F0F11; + box-shadow: 0px 3.999px 48.988px 0px rgba(0, 7, 72, 0.12); + backdrop-filter: blur(7); + box-sizing: border-box; + + .text { + margin-top: 50px; + color: var(--theme-color, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + } + + .view { + margin-top: 32px; + display: flex; + align-items: center; + justify-content: center; + height: 56px; + } +} diff --git a/demo/src/platform/pc/rtc/agent/index.tsx b/demo/src/platform/pc/rtc/agent/index.tsx new file mode 100644 index 00000000..a7fd7944 --- /dev/null +++ b/demo/src/platform/pc/rtc/agent/index.tsx @@ -0,0 +1,34 @@ +"use client" + +import { useAppSelector, useMultibandTrackVolume } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import styles from "./index.module.scss" + +interface AgentProps { + audioTrack?: IMicrophoneAudioTrack +} + +const Agent = (props: AgentProps) => { + const { audioTrack } = props + + const subscribedVolumes = useMultibandTrackVolume(audioTrack, 12); + + return
+
Agent
+
+ +
+
+ +} + + +export default Agent; diff --git a/demo/src/platform/pc/rtc/audioVisualizer/index.module.scss b/demo/src/platform/pc/rtc/audioVisualizer/index.module.scss new file mode 100644 index 00000000..1beae944 --- /dev/null +++ b/demo/src/platform/pc/rtc/audioVisualizer/index.module.scss @@ -0,0 +1,17 @@ +.audioVisualizer { + display: flex; + justify-content: center; + align-items: center; + + + .item {} + + .agent { + background-color: var(--theme-color, #EAECF0); + box-shadow: 0 0 10px var(--theme-color, #EAECF0); + } + + .user { + background-color: var(--Grey-300, #EAECF0); + } +} diff --git a/demo/src/platform/pc/rtc/audioVisualizer/index.tsx b/demo/src/platform/pc/rtc/audioVisualizer/index.tsx new file mode 100644 index 00000000..bc21f554 --- /dev/null +++ b/demo/src/platform/pc/rtc/audioVisualizer/index.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useState, useEffect } from "react" +import styles from "./index.module.scss" + +interface AudioVisualizerProps { + type: "agent" | "user"; + frequencies: Float32Array[]; + gap: number; + barWidth: number; + minBarHeight: number; + maxBarHeight: number + borderRadius: number; +} + + +const AudioVisualizer = (props: AudioVisualizerProps) => { + const { frequencies, gap, barWidth, minBarHeight, maxBarHeight, borderRadius, type } = props; + + const summedFrequencies = frequencies.map((bandFrequencies) => { + const sum = bandFrequencies.reduce((a, b) => a + b, 0) + if (sum <= 0) { + return 0 + } + return Math.sqrt(sum / bandFrequencies.length); + }); + + return
{ + summedFrequencies.map((frequency, index) => { + + const style = { + height: minBarHeight + frequency * (maxBarHeight - minBarHeight) + "px", + borderRadius: borderRadius + "px", + width: barWidth + "px", + transition: + "background-color 0.35s ease-out, transform 0.25s ease-out", + // transform: transform, + } + + return + }) + }
+} + + +export default AudioVisualizer; diff --git a/demo/src/platform/pc/rtc/camSection/camSelect/index.module.scss b/demo/src/platform/pc/rtc/camSection/camSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/demo/src/platform/pc/rtc/camSection/camSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/demo/src/platform/pc/rtc/camSection/camSelect/index.tsx b/demo/src/platform/pc/rtc/camSection/camSelect/index.tsx new file mode 100644 index 00000000..33a5e003 --- /dev/null +++ b/demo/src/platform/pc/rtc/camSection/camSelect/index.tsx @@ -0,0 +1,57 @@ +"use client" + +import AgoraRTC, { ICameraVideoTrack } from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" + +import styles from "./index.module.scss" + +interface CamSelectProps { + videoTrack?: ICameraVideoTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const CamSelect = (props: CamSelectProps) => { + const { videoTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (videoTrack) { + const label = videoTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getCameras().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [videoTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (videoTrack) { + await videoTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default CamSelect diff --git a/demo/src/platform/pc/rtc/camSection/index.module.scss b/demo/src/platform/pc/rtc/camSection/index.module.scss new file mode 100644 index 00000000..28b88e2e --- /dev/null +++ b/demo/src/platform/pc/rtc/camSection/index.module.scss @@ -0,0 +1,54 @@ +.camera { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + .text { + flex: 1 1 auto; + height: 32px; + line-height: 32px; + color: var(--Grey-300, #EAECF0); + font-weight: 500; + letter-spacing: 0.449px; + } + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + .select { + flex: 0 0 auto; + width: 200px; + } + } + + .view { + position: relative; + margin-top: 12px; + min-height: 210px; + height: 210px; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/demo/src/platform/pc/rtc/camSection/index.tsx b/demo/src/platform/pc/rtc/camSection/index.tsx new file mode 100644 index 00000000..99e5392c --- /dev/null +++ b/demo/src/platform/pc/rtc/camSection/index.tsx @@ -0,0 +1,47 @@ +"use client" + +import CamSelect from "./camSelect" +import { CamIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { ICameraVideoTrack } from 'agora-rtc-sdk-ng'; +import { LocalStreamPlayer } from "../streamPlayer" +import { useState, useEffect, useMemo } from 'react'; +import { useSmallScreen } from "@/common" + +interface CamSectionProps { + videoTrack?: ICameraVideoTrack +} + +const CamSection = (props: CamSectionProps) => { + const { videoTrack } = props + const [videoMute, setVideoMute] = useState(false) + const { xs } = useSmallScreen() + + const CamText = useMemo(() => { + return xs ? "CAM" : "CAMERA" + }, [xs]) + + useEffect(() => { + videoTrack?.setMuted(videoMute) + }, [videoTrack, videoMute]) + + const onClickMute = () => { + setVideoMute(!videoMute) + } + + return
+
+ {CamText} + + + + +
+
+ +
+
+} + + +export default CamSection; diff --git a/demo/src/platform/pc/rtc/index.module.scss b/demo/src/platform/pc/rtc/index.module.scss new file mode 100644 index 00000000..b62025c5 --- /dev/null +++ b/demo/src/platform/pc/rtc/index.module.scss @@ -0,0 +1,55 @@ +.rtc { + flex: 0 0 420px; + display: flex; + flex-direction: column; + align-items: flex-start; + flex-shrink: 0; + align-self: stretch; + border-radius: 8px; + border: 1px solid #272A2F; + background: #181A1D; + box-sizing: border-box; + + .header { + display: flex; + height: 42px; + padding: 0px 16px; + align-items: center; + align-self: stretch; + border-bottom: 1px solid #272A2F; + + .text { + flex: 1 1 auto; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + color: var(--Grey-300, #EAECF0); + } + + .voiceSelect { + flex: 0 0 120px; + } + } + + .you { + display: flex; + padding: 24px 16px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 24px; + align-self: stretch; + border-top: 1px solid #272A2F; + + .title { + color: var(--Grey-300, #EAECF0); + font-size: 24px; + font-weight: 600; + line-height: 150%; + letter-spacing: 0.449px; + text-align: center; + } + + + } +} diff --git a/demo/src/platform/pc/rtc/index.tsx b/demo/src/platform/pc/rtc/index.tsx new file mode 100644 index 00000000..1195ca0f --- /dev/null +++ b/demo/src/platform/pc/rtc/index.tsx @@ -0,0 +1,128 @@ +"use client" + +import { ICameraVideoTrack, IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" +import { useAppSelector, useAppDispatch, VOICE_OPTIONS } from "@/common" +import { ITextItem } from "@/types" +import { rtcManager, IUserTracks, IRtcUser } from "@/manager" +import { setRoomConnected, addChatItem, setVoiceType } from "@/store/reducers/global" +import MicSection from "./micSection" +import CamSection from "./camSection" +import Agent from "./agent" +import styles from "./index.module.scss" +import { useRef, useEffect, useState, Fragment } from "react" +import { VoiceIcon } from "@/components/icons" +import CustomSelect from "@/components/customSelect" + +let hasInit = false + +const Rtc = () => { + const dispatch = useAppDispatch() + const options = useAppSelector(state => state.global.options) + const voiceType = useAppSelector(state => state.global.voiceType) + const agentConnected = useAppSelector(state => state.global.agentConnected) + const { userId, channel } = options + const [videoTrack, setVideoTrack] = useState() + const [audioTrack, setAudioTrack] = useState() + const [remoteuser, setRemoteUser] = useState() + + useEffect(() => { + if (!options.channel) { + return + } + if (hasInit) { + return + } + + init() + + return () => { + if (hasInit) { + destory() + } + } + }, [options.channel]) + + + const init = async () => { + console.log("[test] init") + rtcManager.on("localTracksChanged", onLocalTracksChanged) + rtcManager.on("textChanged", onTextChanged) + rtcManager.on("remoteUserChanged", onRemoteUserChanged) + await rtcManager.createTracks() + await rtcManager.join({ + channel, + userId + }) + await rtcManager.publish() + dispatch(setRoomConnected(true)) + hasInit = true + } + + const destory = async () => { + console.log("[test] destory") + rtcManager.off("textChanged", onTextChanged) + rtcManager.off("localTracksChanged", onLocalTracksChanged) + rtcManager.off("remoteUserChanged", onRemoteUserChanged) + await rtcManager.destroy() + dispatch(setRoomConnected(false)) + hasInit = false + } + + const onRemoteUserChanged = (user: IRtcUser) => { + console.log("[test] onRemoteUserChanged", user) + setRemoteUser(user) + } + + const onLocalTracksChanged = (tracks: IUserTracks) => { + console.log("[test] onLocalTracksChanged", tracks) + const { videoTrack, audioTrack } = tracks + if (videoTrack) { + setVideoTrack(videoTrack) + } + if (audioTrack) { + setAudioTrack(audioTrack) + } + } + + const onTextChanged = (text: ITextItem) => { + if (text.dataType == "transcribe") { + const isAgent = Number(text.uid) != Number(userId) + dispatch(addChatItem({ + userId: text.uid, + text: text.text, + type: isAgent ? "agent" : "user", + isFinal: text.isFinal, + time: text.time + })) + } + } + + const onVoiceChange = (value: any) => { + dispatch(setVoiceType(value)) + } + + + return
+
+ Audio & Video + } + options={VOICE_OPTIONS} onChange={onVoiceChange}> +
+ {/* agent */} + + {/* you */} +
+
You
+ {/* microphone */} + + {/* camera */} + +
+
+} + + +export default Rtc; diff --git a/demo/src/platform/pc/rtc/micSection/index.module.scss b/demo/src/platform/pc/rtc/micSection/index.module.scss new file mode 100644 index 00000000..81fffd3d --- /dev/null +++ b/demo/src/platform/pc/rtc/micSection/index.module.scss @@ -0,0 +1,56 @@ +.microphone { + position: relative; + width: 100%; + height: 100%; + box-sizing: border-box; + + .select { + height: 32px; + display: flex; + width: 100%; + justify-content: flex-start; + align-items: center; + + .text { + flex: 1 1 auto; + height: 32px; + line-height: 32px; + color: var(--Grey-300, #EAECF0); + font-weight: 500; + letter-spacing: 0.449px; + } + + .iconWrapper { + flex: 0 0 auto; + margin-right: 12px; + display: flex; + width: 32px; + height: 32px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid #2B2F36; + cursor: pointer; + } + + + } + + .view { + margin-top: 12px; + display: flex; + height: 120px; + padding: 24px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 10px; + align-self: stretch; + border-radius: 6px; + border: 1px solid #272A2F; + background: #1E2024; + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.25); + } +} diff --git a/demo/src/platform/pc/rtc/micSection/index.tsx b/demo/src/platform/pc/rtc/micSection/index.tsx new file mode 100644 index 00000000..6d97f3e2 --- /dev/null +++ b/demo/src/platform/pc/rtc/micSection/index.tsx @@ -0,0 +1,73 @@ +"use client" + +import { useEffect, useMemo, useState } from "react" +import { useMultibandTrackVolume, useSmallScreen } from "@/common" +import AudioVisualizer from "../audioVisualizer" +import { MicIcon } from "@/components/icons" +import styles from "./index.module.scss" +import { IMicrophoneAudioTrack } from 'agora-rtc-sdk-ng'; +import MicSelect from "./micSelect"; + +interface MicSectionProps { + audioTrack?: IMicrophoneAudioTrack +} + +const MicSection = (props: MicSectionProps) => { + const { audioTrack } = props + const [audioMute, setAudioMute] = useState(false) + const [mediaStreamTrack, setMediaStreamTrack] = useState() + const { xs } = useSmallScreen() + + const MicText = useMemo(() => { + return xs ? "MIC" : "MICROPHONE" + }, [xs]) + + useEffect(() => { + audioTrack?.on("track-updated", onAudioTrackupdated) + if (audioTrack) { + setMediaStreamTrack(audioTrack.getMediaStreamTrack()) + } + + return () => { + audioTrack?.off("track-updated", onAudioTrackupdated) + } + }, [audioTrack]) + + useEffect(() => { + audioTrack?.setMuted(audioMute) + }, [audioTrack, audioMute]) + + const subscribedVolumes = useMultibandTrackVolume(mediaStreamTrack, 20); + + const onAudioTrackupdated = (track: MediaStreamTrack) => { + console.log("[test] audio track updated", track) + setMediaStreamTrack(track) + } + + const onClickMute = () => { + setAudioMute(!audioMute) + } + + return
+
+ {MicText} + + + + +
+
+ +
+
+} + + +export default MicSection; diff --git a/demo/src/platform/pc/rtc/micSection/micSelect/index.module.scss b/demo/src/platform/pc/rtc/micSection/micSelect/index.module.scss new file mode 100644 index 00000000..8ca5088b --- /dev/null +++ b/demo/src/platform/pc/rtc/micSection/micSelect/index.module.scss @@ -0,0 +1,4 @@ +.select { + flex: 0 0 200px; + width: 200px; +} diff --git a/demo/src/platform/pc/rtc/micSection/micSelect/index.tsx b/demo/src/platform/pc/rtc/micSection/micSelect/index.tsx new file mode 100644 index 00000000..efc842b5 --- /dev/null +++ b/demo/src/platform/pc/rtc/micSection/micSelect/index.tsx @@ -0,0 +1,58 @@ +"use client" + +import AgoraRTC from "agora-rtc-sdk-ng" +import { useState, useEffect } from "react" +import { Select } from "antd" +import { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng" + +import styles from "./index.module.scss" + +interface MicSelectProps { + audioTrack?: IMicrophoneAudioTrack +} + +interface SelectItem { + label: string + value: string + deviceId: string +} + +const DEFAULT_ITEM: SelectItem = { + label: "Default", + value: "default", + deviceId: "" +} + +const MicSelect = (props: MicSelectProps) => { + const { audioTrack } = props + const [items, setItems] = useState([DEFAULT_ITEM]); + const [value, setValue] = useState("default"); + + useEffect(() => { + if (audioTrack) { + const label = audioTrack?.getTrackLabel(); + setValue(label); + AgoraRTC.getMicrophones().then(arr => { + setItems(arr.map(item => ({ + label: item.label, + value: item.label, + deviceId: item.deviceId + }))); + }); + } + }, [audioTrack]); + + const onChange = async (value: string) => { + const target = items.find(item => item.value === value); + if (target) { + setValue(target.value); + if (audioTrack) { + await audioTrack.setDevice(target.deviceId); + } + } + } + + return +} + +export default MicSelect diff --git a/demo/src/platform/pc/rtc/streamPlayer/index.module.scss b/demo/src/platform/pc/rtc/streamPlayer/index.module.scss new file mode 100644 index 00000000..b1c57c10 --- /dev/null +++ b/demo/src/platform/pc/rtc/streamPlayer/index.module.scss @@ -0,0 +1,6 @@ +.streamPlayer { + position: relative; + width: 100%; + height: 100%; + overflow: hidden; +} diff --git a/demo/src/platform/pc/rtc/streamPlayer/index.tsx b/demo/src/platform/pc/rtc/streamPlayer/index.tsx new file mode 100644 index 00000000..ba78e377 --- /dev/null +++ b/demo/src/platform/pc/rtc/streamPlayer/index.tsx @@ -0,0 +1 @@ +export * from "./localStreamPlayer" diff --git a/demo/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx b/demo/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx new file mode 100644 index 00000000..e3e7f06a --- /dev/null +++ b/demo/src/platform/pc/rtc/streamPlayer/localStreamPlayer.tsx @@ -0,0 +1,46 @@ +"use client" + +import { + ICameraVideoTrack, + IMicrophoneAudioTrack, + IRemoteAudioTrack, + IRemoteVideoTrack, + VideoPlayerConfig, +} from "agora-rtc-sdk-ng" +import { useRef, useState, useLayoutEffect, forwardRef, useEffect, useMemo } from "react" + +import styles from "./index.module.scss" + +interface StreamPlayerProps { + videoTrack?: ICameraVideoTrack + audioTrack?: IMicrophoneAudioTrack + style?: React.CSSProperties + fit?: "cover" | "contain" | "fill" + onClick?: () => void + mute?: boolean +} + +export const LocalStreamPlayer = forwardRef((props: StreamPlayerProps, ref) => { + const { videoTrack, audioTrack, mute = false, style = {}, fit = "cover", onClick = () => { } } = props + const vidDiv = useRef(null) + + useLayoutEffect(() => { + const config = { fit } as VideoPlayerConfig + if (mute) { + videoTrack?.stop() + } else { + if (!videoTrack?.isPlaying) { + videoTrack?.play(vidDiv.current!, config) + } + } + + return () => { + videoTrack?.stop() + } + }, [videoTrack, fit, mute]) + + // local audio track need not to be played + // useLayoutEffect(() => {}, [audioTrack, localAudioMute]) + + return
+}) diff --git a/demo/src/protobuf/SttMessage.js b/demo/src/protobuf/SttMessage.js new file mode 100644 index 00000000..e69de29b diff --git a/demo/src/protobuf/SttMessage.proto b/demo/src/protobuf/SttMessage.proto new file mode 100644 index 00000000..d8e07a54 --- /dev/null +++ b/demo/src/protobuf/SttMessage.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package Agora.SpeechToText; + +option objc_class_prefix = "Stt"; + +option csharp_namespace = "AgoraSTTSample.Protobuf"; + +option java_package = "io.agora.rtc.speech2text"; +option java_outer_classname = "AgoraSpeech2TextProtobuffer"; + +message Text { + int32 vendor = 1; + int32 version = 2; + int32 seqnum = 3; + int64 uid = 4; + int32 flag = 5; + int64 time = 6; + int32 lang = 7; + int32 starttime = 8; + int32 offtime = 9; + repeated Word words = 10; + bool end_of_segment = 11; + int32 duration_ms = 12; + string data_type = 13; // transcribe ,translate + repeated Translation trans = 14; + string culture = 15; +} +message Word { + string text = 1; + int32 start_ms = 2; + int32 duration_ms = 3; + bool is_final = 4; + double confidence = 5; +} +message Translation { + bool is_final = 1; + string lang = 2; // 翻译语言 + repeated string texts = 3; +} diff --git a/demo/src/protobuf/SttMessage_es6.js b/demo/src/protobuf/SttMessage_es6.js new file mode 100644 index 00000000..54188af8 --- /dev/null +++ b/demo/src/protobuf/SttMessage_es6.js @@ -0,0 +1,134 @@ +/* eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars */ +import * as $protobuf from "protobufjs/light" + +const $root = ($protobuf.roots.default || ($protobuf.roots.default = new $protobuf.Root())).addJSON( + { + Agora: { + nested: { + SpeechToText: { + options: { + objc_class_prefix: "Stt", + csharp_namespace: "AgoraSTTSample.Protobuf", + java_package: "io.agora.rtc.speech2text", + java_outer_classname: "AgoraSpeech2TextProtobuffer", + }, + nested: { + Text: { + fields: { + vendor: { + type: "int32", + id: 1, + }, + version: { + type: "int32", + id: 2, + }, + seqnum: { + type: "int32", + id: 3, + }, + uid: { + type: "uint32", + id: 4, + }, + flag: { + type: "int32", + id: 5, + }, + time: { + type: "int64", + id: 6, + }, + lang: { + type: "int32", + id: 7, + }, + starttime: { + type: "int32", + id: 8, + }, + offtime: { + type: "int32", + id: 9, + }, + words: { + rule: "repeated", + type: "Word", + id: 10, + }, + endOfSegment: { + type: "bool", + id: 11, + }, + durationMs: { + type: "int32", + id: 12, + }, + dataType: { + type: "string", + id: 13, + }, + trans: { + rule: "repeated", + type: "Translation", + id: 14, + }, + culture: { + type: "string", + id: 15, + }, + textTs: { + type: "int64", + id: 16, + }, + }, + }, + Word: { + fields: { + text: { + type: "string", + id: 1, + }, + startMs: { + type: "int32", + id: 2, + }, + durationMs: { + type: "int32", + id: 3, + }, + isFinal: { + type: "bool", + id: 4, + }, + confidence: { + type: "double", + id: 5, + }, + }, + }, + Translation: { + fields: { + isFinal: { + type: "bool", + id: 1, + }, + lang: { + type: "string", + id: 2, + }, + texts: { + rule: "repeated", + type: "string", + id: 3, + }, + }, + }, + }, + }, + }, + }, + }, +) + +export { $root as default } diff --git a/demo/src/store/index.ts b/demo/src/store/index.ts new file mode 100644 index 00000000..8c6c1482 --- /dev/null +++ b/demo/src/store/index.ts @@ -0,0 +1,21 @@ +"use client" + +import globalReducer from "./reducers/global" +import { configureStore } from '@reduxjs/toolkit' + +export * from "./provider" + +export const makeStore = () => { + return configureStore({ + reducer: { + global: globalReducer, + }, + devTools: process.env.NODE_ENV !== "production", + }) +} + +// Infer the type of makeStore +export type AppStore = ReturnType +// Infer the `RootState` and `AppDispatch` types from the store itself +export type RootState = ReturnType +export type AppDispatch = AppStore['dispatch'] diff --git a/demo/src/store/provider/index.tsx b/demo/src/store/provider/index.tsx new file mode 100644 index 00000000..f34703b7 --- /dev/null +++ b/demo/src/store/provider/index.tsx @@ -0,0 +1,21 @@ +"use client"; + +import { useRef } from 'react' +import { Provider } from 'react-redux' +import { makeStore, AppStore } from '..' + +export function StoreProvider({ + children +}: { + children: React.ReactNode +}) { + const storeRef = useRef() + + if (!storeRef.current) { + // Create the store instance the first time this renders + storeRef.current = makeStore() + } + + return {children} +} + diff --git a/demo/src/store/reducers/global.ts b/demo/src/store/reducers/global.ts new file mode 100644 index 00000000..b29f0af8 --- /dev/null +++ b/demo/src/store/reducers/global.ts @@ -0,0 +1,105 @@ +import { IOptions, IChatItem, Language, VoiceType } from "@/types" +import { createSlice, PayloadAction } from "@reduxjs/toolkit" +import { DEFAULT_OPTIONS, COLOR_LIST, setOptionsToLocal, genRandomChatList } from "@/common" + +export interface InitialState { + options: IOptions + roomConnected: boolean, + agentConnected: boolean, + themeColor: string, + language: Language + voiceType: VoiceType + chatItems: IChatItem[], + graphName: string +} + +const getInitialState = (): InitialState => { + return { + options: DEFAULT_OPTIONS, + themeColor: COLOR_LIST[0].active, + roomConnected: false, + agentConnected: false, + language: "en-US", + voiceType: "male", + chatItems: [], + graphName: "camera.va.openai.azure" + } +} + +export const globalSlice = createSlice({ + name: "global", + initialState: getInitialState(), + reducers: { + setOptions: (state, action: PayloadAction>) => { + state.options = { ...state.options, ...action.payload } + setOptionsToLocal(state.options) + }, + setThemeColor: (state, action: PayloadAction) => { + state.themeColor = action.payload + document.documentElement.style.setProperty('--theme-color', action.payload); + }, + setRoomConnected: (state, action: PayloadAction) => { + state.roomConnected = action.payload + }, + addChatItem: (state, action: PayloadAction) => { + const { userId, text, isFinal, type, time } = action.payload + const LastFinalIndex = state.chatItems.findLastIndex((el) => { + return el.userId == userId && el.isFinal + }) + const LastNonFinalIndex = state.chatItems.findLastIndex((el) => { + return el.userId == userId && !el.isFinal + }) + let LastFinalItem = state.chatItems[LastFinalIndex] + let LastNonFinalItem = state.chatItems[LastNonFinalIndex] + if (LastFinalItem) { + // has last final Item + if (time <= LastFinalItem.time) { + // discard + console.log("[test] addChatItem, time < last final item, discard!:", text, isFinal, type) + return + } else { + if (LastNonFinalItem) { + console.log("[test] addChatItem, update last item(none final):", text, isFinal, type) + state.chatItems[LastNonFinalIndex] = action.payload + } else { + console.log("[test] addChatItem, add new item:", text, isFinal, type) + state.chatItems.push(action.payload) + } + } + } else { + // no last final Item + if (LastNonFinalItem) { + console.log("[test] addChatItem, update last item(none final):", text, isFinal, type) + state.chatItems[LastNonFinalIndex] = action.payload + } else { + console.log("[test] addChatItem, add new item:", text, isFinal, type) + state.chatItems.push(action.payload) + } + } + state.chatItems.sort((a, b) => a.time - b.time) + }, + setAgentConnected: (state, action: PayloadAction) => { + state.agentConnected = action.payload + }, + setLanguage: (state, action: PayloadAction) => { + state.language = action.payload + }, + setGraphName: (state, action: PayloadAction) => { + state.graphName = action.payload + }, + setVoiceType: (state, action: PayloadAction) => { + state.voiceType = action.payload + }, + reset: (state) => { + Object.assign(state, getInitialState()) + document.documentElement.style.setProperty('--theme-color', COLOR_LIST[0].active); + }, + }, +}) + +export const { reset, setOptions, + setRoomConnected, setAgentConnected, setVoiceType, + addChatItem, setThemeColor, setLanguage, setGraphName } = + globalSlice.actions + +export default globalSlice.reducer diff --git a/demo/src/types/index.ts b/demo/src/types/index.ts new file mode 100644 index 00000000..f5492003 --- /dev/null +++ b/demo/src/types/index.ts @@ -0,0 +1,62 @@ +export type Language = "en-US" | "zh-CN" | "ja-JP" | "ko-KR" +export type VoiceType = "male" | "female" + +export interface ColorItem { + active: string, + default: string +} + + +export interface IOptions { + channel: string, + userName: string, + userId: number +} + + +export interface IChatItem { + userId: number | string, + userName?: string, + text: string + type: "agent" | "user" + isFinal?: boolean + time: number +} + + +export interface ITextItem { + dataType: "transcribe" | "translate" + uid: string + time: number + text: string + isFinal: boolean +} + +export interface GraphOptionItem { + label: string + value: string +} + +export interface LanguageOptionItem { + label: string + value: Language +} + + +export interface VoiceOptionItem { + label: string + value: VoiceType +} + + +export interface OptionType { + value: string; + label: string; +} + + +export interface IPdfData { + fileName: string, + collection: string +} + diff --git a/demo/tsconfig.json b/demo/tsconfig.json new file mode 100644 index 00000000..15dcdd38 --- /dev/null +++ b/demo/tsconfig.json @@ -0,0 +1,40 @@ +{ + "compilerOptions": { + "lib": [ + "dom", + "dom.iterable", + "esnext" + ], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": [ + "./src/*" + ] + } + }, + "include": [ + "svgr.d.ts", + "next-env.d.ts", + "**/*.ts", + "**/*.tsx", + ".next/types/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/docker-compose.yml b/docker-compose.yml index 6ad1a0ef..1793137c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,7 @@ services: - astra_network environment: - AGENT_SERVER_URL=http://astra_agents_dev:8080 + - TEN_DEV_SERVER_URL=http://astra_agents_dev:49483 # use this when you want to run the playground in local development mode # astra_playground_dev: diff --git a/playground/.env b/playground/.env index 5f92b324..8739b9b0 100644 --- a/playground/.env +++ b/playground/.env @@ -1 +1,2 @@ -AGENT_SERVER_URL=http://localhost:8080 \ No newline at end of file +AGENT_SERVER_URL=http://localhost:8080 +TEN_DEV_SERVER_URL=http://localhost:49483 \ No newline at end of file diff --git a/playground/src/app/api/agents/start/route.tsx b/playground/src/app/api/agents/start/route.tsx index 5a7b4440..78a8bc43 100644 --- a/playground/src/app/api/agents/start/route.tsx +++ b/playground/src/app/api/agents/start/route.tsx @@ -1,5 +1,4 @@ import { NextRequest, NextResponse } from 'next/server'; -import { getGraphProperties } from './graph'; /** * Handles the POST request to start an agent. @@ -24,6 +23,7 @@ export async function POST(request: NextRequest) { graph_name, language, voice_type, + properties } = body; // Send a POST request to start the agent @@ -38,7 +38,7 @@ export async function POST(request: NextRequest) { user_uid, graph_name, // Get the graph properties based on the graph name, language, and voice type - properties: getGraphProperties(graph_name, language, voice_type), + properties: properties, }), }); diff --git a/playground/src/app/global.css b/playground/src/app/global.css index f7007287..7a1e1861 100644 --- a/playground/src/app/global.css +++ b/playground/src/app/global.css @@ -8,6 +8,7 @@ html, body { background-color: #0F0F11; font-family: "PingFang SC"; + height: 100%; } a { diff --git a/playground/src/app/page.tsx b/playground/src/app/page.tsx index 1bdcdeda..882aa4bd 100644 --- a/playground/src/app/page.tsx +++ b/playground/src/app/page.tsx @@ -1,14 +1,35 @@ -import LoginCard from "@/components/loginCard" -import styles from "./index.module.scss" +"use client" -export default function Login() { +import AuthInitializer from "@/components/authInitializer" +import { getRandomChannel, getRandomUserId, isMobile, useAppDispatch, useAppSelector } from "@/common" +import dynamic from 'next/dynamic' +import { useEffect, useState } from "react" +import { setOptions } from "@/store/reducers/global" + +const PCEntry = dynamic(() => import('@/platform/pc/entry'), { + ssr: false, +}) + +const MobileEntry = dynamic(() => import('@/platform/mobile/entry'), { + ssr: false, +}) + +export default function Home() { + const dispatch = useAppDispatch() + const [mobile, setMobile] = useState(null); + + + useEffect(() => { + setMobile(isMobile()) + }) return ( -
-
-
-
- -
+ mobile === null ? <> : + + {mobile ? : } + ); } + + + diff --git a/playground/src/common/constant.ts b/playground/src/common/constant.ts index fee0c18e..0fd93c81 100644 --- a/playground/src/common/constant.ts +++ b/playground/src/common/constant.ts @@ -6,7 +6,6 @@ export const DEFAULT_OPTIONS: IOptions = { userName: "", userId: 0 } -export const DESCRIPTION = "This is an AI voice assistant powered by ASTRA.ai framework, Agora, Azure and ChatGPT." export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [ { label: "English", diff --git a/playground/src/common/hooks.ts b/playground/src/common/hooks.ts index 9759fa29..ff1d8050 100644 --- a/playground/src/common/hooks.ts +++ b/playground/src/common/hooks.ts @@ -129,3 +129,16 @@ export const usePrevious = (value: any) => { }; +export const useGraphExtensions = () => { + const graphName = useAppSelector(state => state.global.graphName); + const nodes = useAppSelector(state => state.global.extensions); + const [graphExtensions, setGraphExtensions] = useState>({}); + + useEffect(() => { + if (nodes && nodes[graphName]) { + setGraphExtensions(nodes[graphName]); + } + }, [graphName, nodes]); + + return graphExtensions; +}; \ No newline at end of file diff --git a/playground/src/common/request.ts b/playground/src/common/request.ts index 160cc065..624d5989 100644 --- a/playground/src/common/request.ts +++ b/playground/src/common/request.ts @@ -7,6 +7,7 @@ interface StartRequestConfig { graphName: string, language: Language, voiceType: "male" | "female" + properties: Record } interface GenAgoraDataConfig { @@ -15,7 +16,7 @@ interface GenAgoraDataConfig { } export const apiGenAgoraData = async (config: GenAgoraDataConfig) => { - // the request will be rewrite at next.config.mjs to send to $AGENT_SERVER_URL + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL const url = `/api/token/generate` const { userId, channel } = config const data = { @@ -37,14 +38,15 @@ export const apiGenAgoraData = async (config: GenAgoraDataConfig) => { export const apiStartService = async (config: StartRequestConfig): Promise => { // look at app/api/agents/start/route.tsx for the server-side implementation const url = `/api/agents/start` - const { channel, userId, graphName, language, voiceType } = config + const { channel, userId, graphName, language, voiceType, properties } = config const data = { request_id: genUUID(), channel_name: channel, user_uid: userId, graph_name: graphName, language, - voice_type: voiceType + voice_type: voiceType, + properties, } let resp: any = await fetch(url, { method: "POST", @@ -131,3 +133,42 @@ export const apiPing = async (channel: string) => { resp = (await resp.json()) || {} return resp } + +export const apiGetGraphs = async () => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/dev/v1/graphs` + let resp: any = await fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + } + }) + resp = (await resp.json()) || {} + return resp +} + +export const apiGetExtensionMetadata = async () => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/dev/v1/addons/extensions` + let resp: any = await fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + } + }) + resp = (await resp.json()) || {} + return resp +} + +export const apiGetNodes = async (graphName: string) => { + // the request will be rewrite at middleware.tsx to send to $AGENT_SERVER_URL + const url = `/api/dev/v1/graphs/${graphName}/nodes` + let resp: any = await fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + } + }) + resp = (await resp.json()) || {} + return resp +} \ No newline at end of file diff --git a/playground/src/components/authInitializer/index.tsx b/playground/src/components/authInitializer/index.tsx index 5ef763a1..65336e21 100644 --- a/playground/src/components/authInitializer/index.tsx +++ b/playground/src/components/authInitializer/index.tsx @@ -1,7 +1,7 @@ "use client" import { ReactNode, useEffect } from "react" -import { useAppDispatch, getOptionsFromLocal } from "@/common" +import { useAppDispatch, getOptionsFromLocal, getRandomChannel, getRandomUserId } from "@/common" import { setOptions, reset } from "@/store/reducers/global" interface AuthInitializerProps { @@ -15,9 +15,15 @@ const AuthInitializer = (props: AuthInitializerProps) => { useEffect(() => { if (typeof window !== "undefined") { const options = getOptionsFromLocal() - if (options) { + if (options && options.channel) { dispatch(reset()) dispatch(setOptions(options)) + } else { + dispatch(reset()) + dispatch(setOptions({ + channel: getRandomChannel(), + userId: getRandomUserId(), + })) } } }, [dispatch]) diff --git a/playground/src/middleware.tsx b/playground/src/middleware.tsx index 724e0b4d..d25c18bf 100644 --- a/playground/src/middleware.tsx +++ b/playground/src/middleware.tsx @@ -2,15 +2,19 @@ import { NextRequest, NextResponse } from 'next/server'; -const { AGENT_SERVER_URL } = process.env; +const { AGENT_SERVER_URL, TEN_DEV_SERVER_URL } = process.env; // Check if environment variables are available if (!AGENT_SERVER_URL) { - throw "Environment variables AGENT_SERVER_URL are not available"; + throw "Environment variables AGENT_SERVER_URL are not available"; +} + +if (!TEN_DEV_SERVER_URL) { + throw "Environment variables TEN_DEV_SERVER_URL are not available"; } export function middleware(req: NextRequest) { - const { pathname } = req.nextUrl; + const { pathname } = req.nextUrl; if (pathname.startsWith('/api/agents/')) { if (!pathname.startsWith('/api/agents/start')) { @@ -21,6 +25,8 @@ export function middleware(req: NextRequest) { // console.log(`Rewriting request to ${url.href}`); return NextResponse.rewrite(url); + } else { + return NextResponse.next(); } } else if (pathname.startsWith('/api/vector/')) { @@ -35,6 +41,14 @@ export function middleware(req: NextRequest) { const url = req.nextUrl.clone(); url.href = `${AGENT_SERVER_URL}${pathname.replace('/api/token/', '/token/')}`; + // console.log(`Rewriting request to ${url.href}`); + return NextResponse.rewrite(url); + } else if (pathname.startsWith('/api/dev/')) { + + // Proxy all other documents requests + const url = req.nextUrl.clone(); + url.href = `${TEN_DEV_SERVER_URL}${pathname.replace('/api/dev/', '/api/dev-server/')}`; + // console.log(`Rewriting request to ${url.href}`); return NextResponse.rewrite(url); } else { diff --git a/playground/src/platform/mobile/description/index.tsx b/playground/src/platform/mobile/description/index.tsx index 7473d550..bae88001 100644 --- a/playground/src/platform/mobile/description/index.tsx +++ b/playground/src/platform/mobile/description/index.tsx @@ -1,6 +1,6 @@ import { setAgentConnected } from "@/store/reducers/global" import { - DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, + useAppDispatch, useAppSelector, apiPing, genUUID, apiStartService, apiStopService } from "@/common" import { message } from "antd" @@ -50,7 +50,8 @@ const Description = () => { userId, graphName, language, - voiceType + voiceType, + properties: {} }) const { code, msg } = res || {} if (code != 0) { diff --git a/playground/src/platform/mobile/rtc/agent/index.module.scss b/playground/src/platform/mobile/rtc/agent/index.module.scss index fa3ae2ec..5bebb2bc 100644 --- a/playground/src/platform/mobile/rtc/agent/index.module.scss +++ b/playground/src/platform/mobile/rtc/agent/index.module.scss @@ -1,7 +1,6 @@ .agent { position: relative; display: flex; - height: 292px; padding: 20px 16px; flex-direction: column; justify-content: flex-start; @@ -22,7 +21,6 @@ } .view { - margin-top: 32px; display: flex; align-items: center; justify-content: center; diff --git a/playground/src/platform/mobile/rtc/agent/index.tsx b/playground/src/platform/mobile/rtc/agent/index.tsx index a7fd7944..159f6730 100644 --- a/playground/src/platform/mobile/rtc/agent/index.tsx +++ b/playground/src/platform/mobile/rtc/agent/index.tsx @@ -15,7 +15,6 @@ const Agent = (props: AgentProps) => { const subscribedVolumes = useMultibandTrackVolume(audioTrack, 12); return
-
Agent
{ {/* you */}
-
You
{/* microphone */} {/* camera */} diff --git a/playground/src/platform/pc/chat/index.tsx b/playground/src/platform/pc/chat/index.tsx index 64dea171..3ee02155 100644 --- a/playground/src/platform/pc/chat/index.tsx +++ b/playground/src/platform/pc/chat/index.tsx @@ -7,50 +7,78 @@ import { LANGUAGE_OPTIONS, useAppSelector, GRAPH_OPTIONS, isRagGraph, + apiGetGraphs, + apiGetNodes, + useGraphExtensions, + apiGetExtensionMetadata, } from "@/common" -import { setGraphName, setLanguage } from "@/store/reducers/global" -import { Select, } from 'antd'; +import { setExtensionMetadata, setGraphName, setGraphs, setLanguage, setExtensions } from "@/store/reducers/global" +import { Button, Modal, Select, Tabs, TabsProps, } from 'antd'; import PdfSelect from "@/components/pdfSelect" import styles from "./index.module.scss" - - +import { SettingOutlined } from "@ant-design/icons" +import EditableTable from "./table" const Chat = () => { const dispatch = useAppDispatch() - const chatItems = useAppSelector(state => state.global.chatItems) - const language = useAppSelector(state => state.global.language) + const graphs = useAppSelector(state => state.global.graphs) + const extensions = useAppSelector(state => state.global.extensions) const graphName = useAppSelector(state => state.global.graphName) + const chatItems = useAppSelector(state => state.global.chatItems) const agentConnected = useAppSelector(state => state.global.agentConnected) + const [modal2Open, setModal2Open] = useState(false) + const graphExtensions = useGraphExtensions() + const extensionMetadata = useAppSelector(state => state.global.extensionMetadata) + // const chatItems = genRandomChatList(10) const chatRef = useRef(null) + useEffect(() => { + apiGetGraphs().then((res: any) => { + let graphs = res["data"].map((item: any) => item["name"]) + dispatch(setGraphs(graphs)) + }) + apiGetExtensionMetadata().then((res: any) => { + let metadata = res["data"] + let metadataMap: Record = {} + metadata.forEach((item: any) => { + metadataMap[item["name"]] = item + }) + dispatch(setExtensionMetadata(metadataMap)) + }) + }, []) + + useEffect(() => { + if (!extensions[graphName]) { + apiGetNodes(graphName).then((res: any) => { + let nodes = res["data"] + let nodesMap: Record = {} + nodes.forEach((item: any) => { + nodesMap[item["name"]] = item + }) + dispatch(setExtensions({ graphName, nodesMap })) + }) + } + }, [graphName]) useAutoScroll(chatRef) - - const onLanguageChange = (val: any) => { - dispatch(setLanguage(val)) - } - const onGraphNameChange = (val: any) => { dispatch(setGraphName(val)) } - return
- + {isRagGraph(graphName) ? : null}
@@ -59,6 +87,36 @@ const Chat = () => { return })}
+ setModal2Open(false)} + footer={ + + } + > +

You can adjust extension properties here, the values will be overridden when the agent starts using "Connect." Note that this won't modify the property.json file.

+ { + let node = graphExtensions[key] + let addon = node["addon"] + let metadata = extensionMetadata[addon] + return { + key: node["name"], label: node["name"], children: { + let nodesMap = JSON.parse(JSON.stringify(graphExtensions)) + nodesMap[key]["property"] = data + dispatch(setExtensions({ graphName, nodesMap })) + }} + > + } + })} /> + } diff --git a/playground/src/platform/pc/chat/table/index.tsx b/playground/src/platform/pc/chat/table/index.tsx new file mode 100644 index 00000000..0904d4dc --- /dev/null +++ b/playground/src/platform/pc/chat/table/index.tsx @@ -0,0 +1,168 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { Button, Empty, ConfigProvider, Table, Input, Form, Checkbox } from 'antd'; +import type { ColumnsType } from 'antd/es/table'; + +// Define the data type for the table rows +interface DataType { + key: string; + value: string | number | boolean | null; +} + +// Define the props for the EditableTable component +interface EditableTableProps { + initialData: Record; + onUpdate: (updatedData: Record) => void; + metadata: Record; // Metadata with property types +} + +// Helper to convert values based on type +const convertToType = (value: any, type: string) => { + switch (type) { + case 'int64': + case 'int32': + return parseInt(value, 10); + case 'float64': + return parseFloat(value); + case 'bool': + return value === true || value === 'true'; + case 'string': + return String(value); + default: + return value; + } +}; + +const EditableTable: React.FC = ({ initialData, onUpdate, metadata }) => { + const [dataSource, setDataSource] = useState( + Object.entries(initialData).map(([key, value]) => ({ key, value })) + ); + const [editingKey, setEditingKey] = useState(''); + const [form] = Form.useForm(); + const inputRef = useRef(null); // Ref to manage focus + + // Function to check if the current row is being edited + const isEditing = (record: DataType) => record.key === editingKey; + + // Function to toggle editing on a row + const edit = (record: DataType) => { + form.setFieldsValue({ value: record.value ?? '' }); + setEditingKey(record.key); + }; + + // Function to handle when the value of a non-boolean field is changed + const handleValueChange = async (key: string) => { + try { + const row = await form.validateFields(); + const newData = [...dataSource]; + const index = newData.findIndex((item) => key === item.key); + + if (index > -1) { + const item = newData[index]; + const valueType = metadata[key]?.type || 'string'; + newData.splice(index, 1, { ...item, ...row, value: convertToType(row.value, valueType) }); + setDataSource(newData); + setEditingKey(''); + + // Notify the parent component of the update + const updatedData = Object.fromEntries(newData.map(({ key, value }) => [key, value])); + onUpdate(updatedData); + } + } catch (errInfo) { + console.log('Validation Failed:', errInfo); + } + }; + + // Toggle the checkbox for boolean values directly in the table cell + const handleCheckboxChange = (key: string, checked: boolean) => { + const newData = [...dataSource]; + const index = newData.findIndex((item) => key === item.key); + if (index > -1) { + newData[index].value = checked; // Update the boolean value + setDataSource(newData); + + // Notify the parent component of the update + const updatedData = Object.fromEntries(newData.map(({ key, value }) => [key, value])); + onUpdate(updatedData); + } + }; + + // Auto-focus on the input when entering edit mode + useEffect(() => { + if (editingKey) { + inputRef.current?.focus(); // Focus the input field when editing starts + } + }, [editingKey]); + + // Define columns for the table + const columns: ColumnsType = [ + { + title: 'Key', + dataIndex: 'key', + width: '30%', + key: 'key', + }, + { + title: 'Value', + dataIndex: 'value', + width: '50%', + key: 'value', + render: (_, record: DataType) => { + const valueType = metadata[record.key]?.type || 'string'; + + // Always display the checkbox for boolean values + if (valueType === 'bool') { + return ( + handleCheckboxChange(record.key, e.target.checked)} + /> + ); + } + + // Inline editing for other types (string, number) + const editable = isEditing(record); + return editable ? ( + + handleValueChange(record.key)} // Save on pressing Enter + onBlur={() => handleValueChange(record.key)} // Save on losing focus + /> + + ) : ( +
edit(record)}> + {record.value !== null && record.value !== undefined + ? record.value + : 'Click to edit'} +
// Display placeholder for empty values + ); + }, + }, + ]; + + return ( + ( + + + )} + > +
+ + + + ); +}; + +export default EditableTable; diff --git a/playground/src/platform/pc/description/index.tsx b/playground/src/platform/pc/description/index.tsx index a9a055cd..09d2da5a 100644 --- a/playground/src/platform/pc/description/index.tsx +++ b/playground/src/platform/pc/description/index.tsx @@ -1,7 +1,8 @@ import { setAgentConnected } from "@/store/reducers/global" import { - DESCRIPTION, useAppDispatch, useAppSelector, apiPing, genUUID, - apiStartService, apiStopService + useAppDispatch, useAppSelector, apiPing, genUUID, + apiStartService, apiStopService, + useGraphExtensions } from "@/common" import { Select, Button, message, Upload } from "antd" import { useEffect, useState, MouseEventHandler } from "react" @@ -17,8 +18,9 @@ const Description = () => { const userId = useAppSelector(state => state.global.options.userId) const language = useAppSelector(state => state.global.language) const voiceType = useAppSelector(state => state.global.voiceType) - const graphName = useAppSelector(state => state.global.graphName) const [loading, setLoading] = useState(false) + const graphName = useAppSelector(state => state.global.graphName) + const graphNodes = useGraphExtensions() useEffect(() => { if (channel) { @@ -45,12 +47,18 @@ const Description = () => { message.success("Agent disconnected") stopPing() } else { + let properties: Record = {} + Object.keys(graphNodes).forEach(extensionName => { + properties[extensionName] = {} + properties[extensionName] = graphNodes[extensionName].property + }) const res = await apiStartService({ channel, userId, graphName, language, - voiceType + voiceType, + properties: properties }) const { code, msg } = res || {} if (code != 0) { diff --git a/playground/src/platform/pc/entry/index.module.scss b/playground/src/platform/pc/entry/index.module.scss index f138183f..bb001f27 100644 --- a/playground/src/platform/pc/entry/index.module.scss +++ b/playground/src/platform/pc/entry/index.module.scss @@ -2,16 +2,27 @@ position: relative; height: 100%; box-sizing: border-box; + display: flex; + flex-direction: column; .content { position: relative; padding: 16px; box-sizing: border-box; + display: flex; + height: calc(100% - 64px); + flex-direction: column; .body { margin-top: 16px; display: flex; gap: 24px; + flex-grow: 1; + + .chat { + display: flex; + flex-grow: 1; + } } } -} +} \ No newline at end of file diff --git a/playground/src/platform/pc/entry/index.tsx b/playground/src/platform/pc/entry/index.tsx index e7acd7f1..a7ee7592 100644 --- a/playground/src/platform/pc/entry/index.tsx +++ b/playground/src/platform/pc/entry/index.tsx @@ -11,8 +11,12 @@ const PCEntry = () => {
- - +
+ +
+
+ +
diff --git a/playground/src/platform/pc/rtc/agent/index.module.scss b/playground/src/platform/pc/rtc/agent/index.module.scss index fa3ae2ec..5bebb2bc 100644 --- a/playground/src/platform/pc/rtc/agent/index.module.scss +++ b/playground/src/platform/pc/rtc/agent/index.module.scss @@ -1,7 +1,6 @@ .agent { position: relative; display: flex; - height: 292px; padding: 20px 16px; flex-direction: column; justify-content: flex-start; @@ -22,7 +21,6 @@ } .view { - margin-top: 32px; display: flex; align-items: center; justify-content: center; diff --git a/playground/src/platform/pc/rtc/agent/index.tsx b/playground/src/platform/pc/rtc/agent/index.tsx index a7fd7944..159f6730 100644 --- a/playground/src/platform/pc/rtc/agent/index.tsx +++ b/playground/src/platform/pc/rtc/agent/index.tsx @@ -15,7 +15,6 @@ const Agent = (props: AgentProps) => { const subscribedVolumes = useMultibandTrackVolume(audioTrack, 12); return
-
Agent
{ const dispatch = useAppDispatch() const options = useAppSelector(state => state.global.options) - const voiceType = useAppSelector(state => state.global.voiceType) - const agentConnected = useAppSelector(state => state.global.agentConnected) const { userId, channel } = options const [videoTrack, setVideoTrack] = useState() const [audioTrack, setAudioTrack] = useState() @@ -97,25 +93,15 @@ const Rtc = () => { } } - const onVoiceChange = (value: any) => { - dispatch(setVoiceType(value)) - } - return
Audio & Video - } - options={VOICE_OPTIONS} onChange={onVoiceChange}>
{/* agent */} {/* you */}
-
You
{/* microphone */} {/* camera */} diff --git a/playground/src/store/reducers/global.ts b/playground/src/store/reducers/global.ts index b29f0af8..dbdbff3b 100644 --- a/playground/src/store/reducers/global.ts +++ b/playground/src/store/reducers/global.ts @@ -10,7 +10,10 @@ export interface InitialState { language: Language voiceType: VoiceType chatItems: IChatItem[], - graphName: string + graphName: string, + graphs: string[], + extensions: Record, + extensionMetadata: Record } const getInitialState = (): InitialState => { @@ -22,7 +25,10 @@ const getInitialState = (): InitialState => { language: "en-US", voiceType: "male", chatItems: [], - graphName: "camera.va.openai.azure" + graphName: "camera.va.openai.azure", + graphs: [], + extensions: {}, + extensionMetadata: {}, } } @@ -87,6 +93,16 @@ export const globalSlice = createSlice({ setGraphName: (state, action: PayloadAction) => { state.graphName = action.payload }, + setGraphs: (state, action: PayloadAction) => { + state.graphs = action.payload + }, + setExtensions: (state, action: PayloadAction>) => { + let { graphName, nodesMap } = action.payload + state.extensions[graphName] = nodesMap + }, + setExtensionMetadata: (state, action: PayloadAction>) => { + state.extensionMetadata = action.payload + }, setVoiceType: (state, action: PayloadAction) => { state.voiceType = action.payload }, @@ -99,7 +115,7 @@ export const globalSlice = createSlice({ export const { reset, setOptions, setRoomConnected, setAgentConnected, setVoiceType, - addChatItem, setThemeColor, setLanguage, setGraphName } = + addChatItem, setThemeColor, setLanguage, setGraphName, setGraphs, setExtensions, setExtensionMetadata } = globalSlice.actions export default globalSlice.reducer