diff --git a/.github/workflows/docker-image-dev.yml b/.github/workflows/docker-image-dev.yml new file mode 100644 index 000000000000..09e8d52d8969 --- /dev/null +++ b/.github/workflows/docker-image-dev.yml @@ -0,0 +1,152 @@ +name: Publish Docker image (dev) + +# Builds the full image (Dockerfile): frontend (web/default + classic) is +# built with bun and embedded into the Go binary, so a single port (:3000) +# serves both the API and the frontend. Pushed to the GitHub Container +# Registry (ghcr.io) under this repo's owner namespace. Authenticates with +# the built-in GITHUB_TOKEN, so no Docker Hub secrets are required. +# +# Each architecture is built natively in parallel (no QEMU emulation), then +# merged into a single multi-arch manifest — much faster than a single +# emulated multi-arch build. +# +# After the first publish, set the package to "public" at +# https://github.com/users//packages/container/new-api/settings +# so it can be pulled anonymously. + +on: + push: + branches: + - feat/image-aware-model-routing + paths-ignore: + - '*.md' + - 'docs/**' + workflow_dispatch: + inputs: + name: + description: "reason" + required: false + +jobs: + build_single_arch: + name: Build & push (${{ matrix.arch }}) [native] + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + platform: linux/amd64 + runner: ubuntu-latest + - arch: arm64 + platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + permissions: + packages: write + contents: read + steps: + - name: Check out (shallow) + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Resolve GHCR image name + id: ghcr + run: | + OWNER_LC=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + echo "image=ghcr.io/${OWNER_LC}/new-api" >> $GITHUB_OUTPUT + + - name: Determine dev version + id: version + run: | + SHORT_SHA=$(git rev-parse --short HEAD) + echo "versioned=dev-${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "Publishing dev image for ${{ matrix.arch }}: dev, dev-${SHORT_SHA}" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build & push single-arch + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: ${{ matrix.platform }} + push: true + tags: | + ${{ steps.ghcr.outputs.image }}:dev-${{ matrix.arch }} + ${{ steps.ghcr.outputs.image }}:${{ steps.version.outputs.versioned }}-${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Output digest + run: | + echo "### Dev image (${{ matrix.arch }})" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "${{ steps.ghcr.outputs.image }}:dev-${{ matrix.arch }}" >> $GITHUB_STEP_SUMMARY + echo "${{ steps.build.outputs.digest }}" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + create_manifests: + name: Create multi-arch manifest + needs: [build_single_arch] + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Check out (shallow) + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Resolve GHCR image name + id: ghcr + run: | + OWNER_LC=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + echo "image=ghcr.io/${OWNER_LC}/new-api" >> $GITHUB_OUTPUT + + - name: Determine dev version + id: version + run: | + SHORT_SHA=$(git rev-parse --short HEAD) + echo "versioned=dev-${SHORT_SHA}" >> $GITHUB_OUTPUT + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create & push manifest (dev) + run: | + docker buildx imagetools create \ + -t ${{ steps.ghcr.outputs.image }}:dev \ + ${{ steps.ghcr.outputs.image }}:dev-amd64 \ + ${{ steps.ghcr.outputs.image }}:dev-arm64 + + - name: Create & push manifest (versioned) + run: | + docker buildx imagetools create \ + -t ${{ steps.ghcr.outputs.image }}:${{ steps.version.outputs.versioned }} \ + ${{ steps.ghcr.outputs.image }}:${{ steps.version.outputs.versioned }}-amd64 \ + ${{ steps.ghcr.outputs.image }}:${{ steps.version.outputs.versioned }}-arm64 + + - name: Output manifest digest + run: | + echo "### Multi-arch Manifest (dev)" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker buildx imagetools inspect ${{ steps.ghcr.outputs.image }}:dev >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY diff --git a/constant/context_key.go b/constant/context_key.go index b856bc3dda14..cc8358ab4e5b 100644 --- a/constant/context_key.go +++ b/constant/context_key.go @@ -19,6 +19,7 @@ const ( ContextKeyTokenModelLimitEnabled ContextKey = "token_model_limit_enabled" ContextKeyTokenModelLimit ContextKey = "token_model_limit" ContextKeyTokenCrossGroupRetry ContextKey = "token_cross_group_retry" + ContextKeyTokenModelRouteNotify ContextKey = "token_model_route_notify" /* channel related keys */ ContextKeyChannelId ContextKey = "channel_id" @@ -42,6 +43,10 @@ const ( ContextKeyAutoGroupIndex ContextKey = "auto_group_index" ContextKeyAutoGroupRetryIndex ContextKey = "auto_group_retry_index" + /* image-aware routing keys */ + ContextKeyImageAwareEntryModel ContextKey = "image_aware_entry_model" + ContextKeyImageAwareHasImage ContextKey = "image_aware_has_image" + /* user related keys */ ContextKeyUserId ContextKey = "id" ContextKeyUserSetting ContextKey = "user_setting" diff --git a/controller/token.go b/controller/token.go index 836e9b2952ac..9f088a048ab5 100644 --- a/controller/token.go +++ b/controller/token.go @@ -221,6 +221,7 @@ func AddToken(c *gin.Context) { AllowIps: token.AllowIps, Group: token.Group, CrossGroupRetry: token.CrossGroupRetry, + ModelRouteNotify: token.ModelRouteNotify, } err = cleanToken.Insert() if err != nil { @@ -299,6 +300,7 @@ func UpdateToken(c *gin.Context) { cleanToken.AllowIps = token.AllowIps cleanToken.Group = token.Group cleanToken.CrossGroupRetry = token.CrossGroupRetry + cleanToken.ModelRouteNotify = token.ModelRouteNotify } err = cleanToken.Update() if err != nil { diff --git a/docker-compose.deploy.yml b/docker-compose.deploy.yml new file mode 100644 index 000000000000..c5e8818f7997 --- /dev/null +++ b/docker-compose.deploy.yml @@ -0,0 +1,71 @@ +# Deployment compose - pulls the dev image published to the GitHub Container +# Registry by CI (.github/workflows/docker-image-dev.yml). The image is built +# from Dockerfile.dev (backend-only, frontend served by the image placeholder). +# +# If the GHCR package is private, log in first: +# echo $GITHUB_TOKEN | docker login ghcr.io -u --password-stdin +# +# Usage: +# 1. docker compose -f docker-compose.deploy.yml up -d +# 2. Open http://localhost:3000 +# +# Stop: +# docker compose -f docker-compose.deploy.yml down +# +# Reset data: +# docker compose -f docker-compose.deploy.yml down -v + +services: + new-api: + image: ghcr.io/gentle-lijie/new-api:dev + container_name: new-api-deploy + restart: unless-stopped + ports: + - "3000:3000" + volumes: + - deploy_data:/data + environment: + - SQL_DSN=postgresql://root:123456@postgres:5432/new-api + - REDIS_CONN_STRING=redis://redis + - TZ=Asia/Shanghai + - BATCH_UPDATE_ENABLED=true + depends_on: + redis: + condition: service_started + postgres: + condition: service_healthy + networks: + - deploy-network + + redis: + image: redis:7-alpine + container_name: new-api-deploy-redis + restart: unless-stopped + networks: + - deploy-network + + postgres: + image: postgres:15-alpine + container_name: new-api-deploy-pg + restart: unless-stopped + environment: + POSTGRES_USER: root + POSTGRES_PASSWORD: 123456 + POSTGRES_DB: new-api + volumes: + - deploy_pg_data:/var/lib/postgresql/data + networks: + - deploy-network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U root -d new-api"] + interval: 5s + timeout: 3s + retries: 5 + +volumes: + deploy_data: + deploy_pg_data: + +networks: + deploy-network: + driver: bridge diff --git a/docs/PR-description.md b/docs/PR-description.md new file mode 100644 index 000000000000..d90f4415860f --- /dev/null +++ b/docs/PR-description.md @@ -0,0 +1,46 @@ +> [!IMPORTANT] +> 本 PR 由 AI 辅助生成（git user `GentleLijie` 非历史核心开发者），已人工整理描述如下。 + +## 📝 变更描述 / Description + +新增「图片感知模型路由」：配置一个**虚拟入口模型名**（如 `auto-coder`），网关在 distributor 选渠道之前，解析请求体检测**最后一条 `role=user` 消息**是否含图片（同时支持 OpenAI `image_url` 与 Claude `image` 两种 content part），据此把模型名改写为配置好的**视觉模型**或**编程模型**。改写发生在渠道选择之前，因此真实模型名会参与渠道选择、亲和性、计费与重试。 + +由于网关每个请求无状态，“图片轮走视觉模型、后续纯文本轮回到编程模型”由客户端每轮携带的完整对话历史天然完成，网关无需存任何状态——仅看当前轮最后一条 user 消息，避免历史残留图片误触发。 + +可观测性： +- 响应头注入 `X-Routed-Model` / `X-Route-Entry-Model` / `X-Route-Reason` +- Token 级 `ModelRouteNotify` 开关（默认对新 token 开启）控制是否在**响应体内**注入提示文本（如 `> [Route: auto-coder → glm-4.6v (image detected)]`），覆盖 OpenAI/Claude 客户端格式 × 流式/非流式 +- 日志 `Log.Other` 写入 `image_aware_entry_model`，用量日志 Model 列显示相机图标 + 入口模型 Popover + +管理后台提供向导式 Drawer 配置路由规则（入口模型 + 视觉/编程模型下拉选择）。 + +## 🚀 变更类型 / Type of change +- [x] ✨ 新功能 (New feature) + +## 🔗 关联任务 / Related Issue +- 无对应 Issue + +## ✅ 提交前检查项 / Checklist +- [x] **非重复提交:** 已确认无重复 PR +- [x] **变更理解:** 见上方描述 +- [x] **范围聚焦:** 排除了无关的 `__root.tsx`（devtools 注释）与 `pnpm-lock.yaml`（npm 误生成） +- [x] **本地验证:** 后端 `go build ./...` 通过；`go test ./middleware/`（图片检测表驱动单测 12 例）通过；前端 `tsc -b` 涉及文件无类型错误 +- [x] **安全合规:** 无敏感凭据；JSON 统一走 `common.*`；配置仅写 options 表字符串，三库兼容 + +> 注：checklist 中「人工确认」项因本 PR 为 AI 辅助生成，未勾选，已在此如实标注。 + +## 📸 运行证明 / Proof of Work + +带图请求被正确路由到视觉模型，纯文本请求回到编程模型（`record consume log` 的 `model_name` 字段）： +```text +model_name=glm-4.6v prompt_tokens=41355 image_aware_entry_model=auto-coder // 含图 → 视觉模型 +model_name=glm-5 prompt_tokens=79 image_aware_entry_model=auto-coder // 纯文本 → 编程模型 +model_name=glm-5 prompt_tokens=41188 image_aware_entry_model=auto-coder // 后续纯文本轮，仍回编程模型 +``` + +路由决策日志（distributor，每请求一次，不刷屏）： +```text +image_aware_routing: entry=auto-coder has_image=true -> routed=glm-4.6v notify=true +``` + +Token 开启 `ModelRouteNotify` 后，响应流首个内容 delta 前置提示文本，客户端可见 `> [Route: auto-coder → glm-4.6v (image detected)]`。 diff --git a/docs/image-aware-routing.md b/docs/image-aware-routing.md new file mode 100644 index 000000000000..a465a9f91711 --- /dev/null +++ b/docs/image-aware-routing.md @@ -0,0 +1,118 @@ +# Image-Aware Model Routing（图片感知模型路由） + +## 功能概述 + +允许你配置一个**虚拟入口模型名**（如 `auto-coder`），客户端统一向这个名字发请求，网关自动根据当前轮请求内容决定路由目标： + +- 最后一条 user 消息**包含图片** → 路由到配置好的**视觉模型**（如 `gpt-4o`） +- 最后一条 user 消息**不含图片** → 路由到配置好的**编程模型**（如 `claude-sonnet-4`） + +整个过程对客户端透明，客户端只需发同一个模型名，无需感知切换。 + +## 配置方式 + +在管理后台进入：**运维设置 → Image-Aware Model Routing** + +在 JSON textarea 中填入路由规则，格式如下： + +```json +{ + "auto-coder": { + "vision_model": "gpt-4o", + "coding_model": "claude-sonnet-4" + } +} +``` + +- **key**（`auto-coder`）：客户端请求时发送的模型名，即虚拟入口名，可自定义 +- **`vision_model`**：检测到图片时路由到的真实模型名，需在该用户 group 下有对应渠道 +- **`coding_model`**：无图片时路由到的真实模型名，需在该用户 group 下有对应渠道 + +保存后立即生效，无需重启。 + +**关闭功能**：将内容改为 `{}` 保存即可。 + +## 多入口示例 + +可同时配置多个虚拟模型名，互不干扰： + +```json +{ + "auto-coder": { + "vision_model": "gpt-4o", + "coding_model": "claude-sonnet-4" + }, + "auto-writer": { + "vision_model": "gpt-4o", + "coding_model": "gpt-4-turbo" + } +} +``` + +## 使用方式（客户端） + +以 OpenAI 兼容 API 为例，将 `model` 字段设为配置好的入口名即可： + +```bash +# 发送带图片的请求（会路由到 vision_model） +curl https://your-newapi-host/v1/chat/completions \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "auto-coder", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "这张截图里的代码有什么问题？"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}} + ] + } + ] + }' + +# 发送纯文本请求（会路由到 coding_model） +curl https://your-newapi-host/v1/chat/completions \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "auto-coder", + "messages": [ + {"role": "user", "content": "用 Python 实现一个二分查找"} + ] + }' +``` + +在 IDE 插件（Cursor、Cline、Continue 等）中，将模型名设置为 `auto-coder` 即可。 + +## 多轮对话行为 + +网关只检测**当前请求最后一条 role=user 的消息**，历史消息不影响路由决策： + +| 轮次 | 最后一条 user 消息 | 路由目标 | +|------|-------------------|---------| +| 第 1 轮 | 含图片（截图分析） | `vision_model` | +| 第 2 轮 | 纯文本（根据分析写代码） | `coding_model` | +| 第 3 轮 | 纯文本（优化代码） | `coding_model` | +| 第 4 轮 | 含新图片（另一张截图） | `vision_model` | + +第 2 轮的 coding model 能看到第 1 轮 vision model 的输出，因为客户端会把完整对话历史随请求发送，网关无需存储任何状态。 + +## 渠道配置前提 + +虚拟入口名（`auto-coder`）**不需要**对应任何真实渠道。 + +但 `vision_model` 和 `coding_model` 指定的模型名**必须**在请求用户所在 group 下有可用渠道，否则会返回「找不到可用渠道」错误。配置路由规则前，请先确认： + +1. 目标模型已在「渠道管理」中添加并启用 +2. 对应渠道已在用户所在分组的 ability 表中注册（正常添加渠道后自动完成） + +## Token 权限 + +如果你使用了 **Token 模型限制**（在 Token 配置里勾选了允许的模型），只需把**虚拟入口名**（`auto-coder`）加入允许列表即可，无需额外添加 `vision_model` / `coding_model`。路由后的真实模型作为网关内部细节处理。 + +## 日志与计费 + +- **计费**按实际使用的真实模型（`gpt-4o` 或 `claude-sonnet-4`）计算，不按虚拟入口名 +- **日志**中显示的模型名为真实模型名 +- 如需在日志中追溯入口名：用量日志的 Model 列会显示入口模型（含相机图标与入口名 Popover），同时 `image_aware_entry_model` 仍会记录在上下文元数据中 diff --git a/middleware/auth.go b/middleware/auth.go index 5f2ed4899d4c..a76d82482b06 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -436,6 +436,7 @@ func SetupContextForToken(c *gin.Context, token *model.Token, parts ...string) e } common.SetContextKey(c, constant.ContextKeyTokenGroup, token.Group) common.SetContextKey(c, constant.ContextKeyTokenCrossGroupRetry, token.CrossGroupRetry) + common.SetContextKey(c, constant.ContextKeyTokenModelRouteNotify, token.ModelRouteNotify) if len(parts) > 1 { if model.IsAdmin(token.UserId) { c.Set("specific_channel_id", parts[1]) diff --git a/middleware/distributor.go b/middleware/distributor.go index cf5caa06d513..75a9418c0a88 100644 --- a/middleware/distributor.go +++ b/middleware/distributor.go @@ -77,6 +77,8 @@ func Distribute() func(c *gin.Context) { } if shouldSelectChannel { + // image-aware：虚拟入口模型按最后一条 user 消息是否含图片改写为视觉/编程模型，须在选渠道之前完成。 + ApplyImageAwareRouting(c, modelRequest) if modelRequest.Model == "" { abortWithOpenAiMessage(c, http.StatusBadRequest, i18n.T(c, i18n.MsgDistributorModelNameRequired)) return diff --git a/middleware/image_aware_routing.go b/middleware/image_aware_routing.go new file mode 100644 index 000000000000..70543e7b8ba6 --- /dev/null +++ b/middleware/image_aware_routing.go @@ -0,0 +1,99 @@ +package middleware + +import ( + "fmt" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/setting/operation_setting" + + "github.com/gin-gonic/gin" + "github.com/tidwall/gjson" +) + +const contentTypeImageURL = "image_url" +const contentTypeImage = "image" + +// ApplyImageAwareRouting 在选渠道前调用：入口模型按最后一条 user 消息是否含图片改写为视觉/编程模型， +// 改写后的真实模型名会参与渠道选择、亲和性、计费与重试。 +func ApplyImageAwareRouting(c *gin.Context, modelRequest *ModelRequest) bool { + if modelRequest == nil { + return false + } + rule, ok := operation_setting.GetImageAwareRouteRule(modelRequest.Model) + if !ok { + return false + } + + hasImage, err := detectImageInLastUserMessage(c) + if err != nil { + common.SysLog("image_aware_routing: failed to parse request body: " + err.Error()) + hasImage = false + } + + entryModel := modelRequest.Model + if hasImage { + modelRequest.Model = rule.VisionModel + } else { + modelRequest.Model = rule.CodingModel + } + notify := common.GetContextKeyBool(c, constant.ContextKeyTokenModelRouteNotify) + common.SysLog(fmt.Sprintf("image_aware_routing: entry=%s has_image=%v -> routed=%s notify=%v", entryModel, hasImage, modelRequest.Model, notify)) + common.SetContextKey(c, constant.ContextKeyImageAwareEntryModel, entryModel) + common.SetContextKey(c, constant.ContextKeyImageAwareHasImage, hasImage) + return true +} + +// detectImageInLastUserMessage 仅看最后一条 user 消息，使后续纯文本轮即使历史残留图片也能回到编程模型。 +func detectImageInLastUserMessage(c *gin.Context) (bool, error) { + storage, err := common.GetBodyStorage(c) + if err != nil { + return false, err + } + requestBody, err := storage.Bytes() + if err != nil { + return false, err + } + if _, err := storage.Seek(0, 0); err != nil { + return false, err + } + return hasImageInLastUserMessage(requestBody), nil +} + +func hasImageInLastUserMessage(requestBody []byte) bool { + if !gjson.ValidBytes(requestBody) { + return false + } + + messages := gjson.GetBytes(requestBody, "messages") + if !messages.IsArray() { + return false + } + + var lastUserContent gjson.Result + found := false + messages.ForEach(func(_, message gjson.Result) bool { + if message.Get("role").String() == "user" { + lastUserContent = message.Get("content") + found = true + } + return true + }) + if !found { + return false + } + + if !lastUserContent.IsArray() { + return false + } + hasImage := false + lastUserContent.ForEach(func(_, part gjson.Result) bool { + partType := part.Get("type").String() + if partType == contentTypeImageURL || partType == contentTypeImage { + hasImage = true + return false + } + return true + }) + return hasImage +} diff --git a/middleware/image_aware_routing_test.go b/middleware/image_aware_routing_test.go new file mode 100644 index 000000000000..0285b4593cf2 --- /dev/null +++ b/middleware/image_aware_routing_test.go @@ -0,0 +1,83 @@ +package middleware + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHasImageInLastUserMessage(t *testing.T) { + tests := []struct { + name string + body string + want bool + }{ + { + name: "plain string content, no image", + body: `{"model":"auto-coder","messages":[{"role":"user","content":"hello"}]}`, + want: false, + }, + { + name: "last user message has image_url", + body: `{"model":"auto-coder","messages":[{"role":"user","content":[{"type":"text","text":"what is this"},{"type":"image_url","image_url":{"url":"data:image/png;base64,xxx"}}]}]}`, + want: true, + }, + { + name: "history has image but last user message is text-only", + body: `{"model":"auto-coder","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"x"}}]},{"role":"assistant","content":"a cat"},{"role":"user","content":"now write the code"}]}`, + want: false, + }, + { + name: "last user text-only array content", + body: `{"model":"auto-coder","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + want: false, + }, + { + name: "no user message", + body: `{"model":"auto-coder","messages":[{"role":"system","content":"sys"},{"role":"assistant","content":"hi"}]}`, + want: false, + }, + { + name: "empty messages", + body: `{"model":"auto-coder","messages":[]}`, + want: false, + }, + { + name: "messages missing", + body: `{"model":"auto-coder"}`, + want: false, + }, + { + name: "invalid json", + body: `{not json`, + want: false, + }, + { + name: "image present in a non-last user turn but final user is text", + body: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"a"}}]},{"role":"user","content":"follow up text"}]}`, + want: false, + }, + { + name: "image in the very last user message after several turns", + body: `{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"},{"role":"user","content":"another"},{"role":"assistant","content":"reply"},{"role":"user","content":[{"type":"text","text":"look"},{"type":"image_url","image_url":{"url":"b"}}]}]}`, + want: true, + }, + { + name: "Claude format: last user message has type=image", + body: `{"messages":[{"role":"user","content":[{"type":"text","text":"analyze"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo..."}}]}]}`, + want: true, + }, + { + name: "Claude format: text-only no image", + body: `{"messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]}`, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := hasImageInLastUserMessage([]byte(tt.body)) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/model/option.go b/model/option.go index ed1af72ebb12..4f0f87bff76f 100644 --- a/model/option.go +++ b/model/option.go @@ -173,6 +173,7 @@ func InitOptionMap() { common.OptionMap["AutomaticDisableStatusCodes"] = operation_setting.AutomaticDisableStatusCodesToString() common.OptionMap["AutomaticRetryStatusCodes"] = operation_setting.AutomaticRetryStatusCodesToString() common.OptionMap["ExposeRatioEnabled"] = strconv.FormatBool(ratio_setting.IsExposeRatioEnabled()) + common.OptionMap["ImageAwareModelRouting"] = operation_setting.ImageAwareModelRouting2JSONString() // 自动添加所有注册的模型配置 modelConfigs := config.GlobalConfig.ExportAllConfigs() @@ -542,6 +543,8 @@ func updateOptionMap(key string, value string) (err error) { err = ratio_setting.UpdateAudioCompletionRatioByJSONString(value) case "TopUpLink": common.TopUpLink = value + case "ImageAwareModelRouting": + err = operation_setting.UpdateImageAwareModelRoutingByJSONString(value) //case "ChatLink": // common.ChatLink = value //case "ChatLink2": diff --git a/model/token.go b/model/token.go index ab841f6054ef..3121424a21ff 100644 --- a/model/token.go +++ b/model/token.go @@ -28,6 +28,7 @@ type Token struct { UsedQuota int `json:"used_quota" gorm:"default:0"` // used quota Group string `json:"group" gorm:"default:''"` CrossGroupRetry bool `json:"cross_group_retry"` // 跨分组重试，仅auto分组有效 + ModelRouteNotify bool `json:"model_route_notify" gorm:"default:false"` DeletedAt gorm.DeletedAt `gorm:"index"` } @@ -295,7 +296,7 @@ func (token *Token) Update() (err error) { } }() err = DB.Model(token).Select("name", "status", "expired_time", "remain_quota", "unlimited_quota", - "model_limits_enabled", "model_limits", "allow_ips", "group", "cross_group_retry").Updates(token).Error + "model_limits_enabled", "model_limits", "allow_ips", "group", "cross_group_retry", "model_route_notify").Updates(token).Error return err } diff --git a/relay/channel/claude/relay-claude.go b/relay/channel/claude/relay-claude.go index 0ba31b1b9baa..754021d4e63f 100644 --- a/relay/channel/claude/relay-claude.go +++ b/relay/channel/claude/relay-claude.go @@ -584,12 +584,13 @@ func ResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.OpenAITextRe } type ClaudeResponseInfo struct { - ResponseId string - Created int64 - Model string - ResponseText strings.Builder - Usage *dto.Usage - Done bool + ResponseId string + Created int64 + Model string + ResponseText strings.Builder + Usage *dto.Usage + Done bool + RouteHintInjected bool } func cacheCreationTokensForOpenAIUsage(usage *dto.Usage) int { @@ -816,6 +817,21 @@ func HandleStreamResponseData(c *gin.Context, info *relaycommon.RelayInfo, claud data = patchClaudeMessageDeltaUsageData(data, buildMessageDeltaPatchUsage(&claudeResponse, claudeInfo)) } } + if !claudeInfo.RouteHintInjected && claudeResponse.Type == "content_block_delta" && + claudeResponse.Delta != nil && claudeResponse.Delta.Type == "text_delta" { + if hint := helper.RouteHint(c, info); hint != "" { + hintText := hint + hintResp := &dto.ClaudeResponse{ + Type: "content_block_delta", + Index: claudeResponse.Index, + Delta: &dto.ClaudeMediaMessage{Type: "text_delta", Text: &hintText}, + } + if hintData, mErr := common.Marshal(hintResp); mErr == nil { + helper.ClaudeChunkData(c, *hintResp, string(hintData)) + } + claudeInfo.RouteHintInjected = true + } + } helper.ClaudeChunkData(c, claudeResponse, data) } else if info.RelayFormat == types.RelayFormatOpenAI { response := StreamResponseClaude2OpenAI(&claudeResponse) @@ -879,6 +895,23 @@ func ClaudeStreamHandler(c *gin.Context, resp *http.Response, info *relaycommon. Usage: &dto.Usage{}, } var err *types.NewAPIError + if info.RelayFormat == types.RelayFormatOpenAI { + if hint := helper.RouteHint(c, info); hint != "" { + hintChunk := &dto.ChatCompletionsStreamResponse{ + Id: c.GetString("request_id"), + Object: "chat.completion.chunk", + Created: common.GetTimestamp(), + Model: info.UpstreamModelName, + Choices: []dto.ChatCompletionsStreamResponseChoice{ + {Index: 0, Delta: dto.ChatCompletionsStreamResponseChoiceDelta{Role: "assistant"}}, + }, + } + hintChunk.Choices[0].Delta.SetContentString(hint) + if e := helper.ObjectData(c, hintChunk); e != nil { + logger.LogError(c, "send route hint chunk failed: "+e.Error()) + } + } + } helper.StreamScannerHandler(c, resp, info, func(data string, sr *helper.StreamResult) { err = HandleStreamResponseData(c, info, claudeInfo, data) if err != nil { @@ -917,16 +950,30 @@ func HandleClaudeResponseData(c *gin.Context, info *relaycommon.RelayInfo, claud claudeInfo.Usage.ClaudeCacheCreation1hTokens = claudeResponse.Usage.GetCacheCreation1hTokens() } var responseData []byte + hint := helper.RouteHint(c, info) switch info.RelayFormat { case types.RelayFormatOpenAI: openaiResponse := ResponseClaude2OpenAI(&claudeResponse) openaiResponse.Usage = buildOpenAIStyleUsageFromClaudeUsage(claudeInfo.Usage) - responseData, err = json.Marshal(openaiResponse) + if hint != "" && len(openaiResponse.Choices) > 0 && openaiResponse.Choices[0].Message.IsStringContent() { + msg := &openaiResponse.Choices[0].Message + msg.SetStringContent(hint + msg.StringContent()) + } + responseData, err = common.Marshal(openaiResponse) if err != nil { return types.NewError(err, types.ErrorCodeBadResponseBody) } case types.RelayFormatClaude: - responseData = data + if hint != "" && len(claudeResponse.Content) > 0 && claudeResponse.Content[0].Type == "text" && claudeResponse.Content[0].Text != nil { + prepended := hint + *claudeResponse.Content[0].Text + claudeResponse.Content[0].Text = &prepended + responseData, err = common.Marshal(claudeResponse) + if err != nil { + responseData = data + } + } else { + responseData = data + } } if claudeResponse.Usage != nil && claudeResponse.Usage.ServerToolUse != nil && claudeResponse.Usage.ServerToolUse.WebSearchRequests > 0 { diff --git a/relay/channel/openai/helper.go b/relay/channel/openai/helper.go index 1a01d06da6dc..f26e3f7c24db 100644 --- a/relay/channel/openai/helper.go +++ b/relay/channel/openai/helper.go @@ -21,6 +21,8 @@ import ( func HandleStreamFormat(c *gin.Context, info *relaycommon.RelayInfo, data string, forceFormat bool, thinkToContent bool) error { info.SendResponseCount++ + data = injectRouteHintIfNeeded(c, info, data) + switch info.RelayFormat { case types.RelayFormatOpenAI: return sendStreamData(c, info, data, forceFormat, thinkToContent) @@ -32,6 +34,38 @@ func HandleStreamFormat(c *gin.Context, info *relaycommon.RelayInfo, data string return nil } +// injectRouteHintIfNeeded 在首个含文本内容的 stream chunk 前置路由提示，仅注入一次。 +func injectRouteHintIfNeeded(c *gin.Context, info *relaycommon.RelayInfo, data string) string { + if info.RouteHintInjected || data == "" { + return data + } + hint := helper.RouteHint(c, info) + if hint == "" { + return data + } + var streamResponse dto.ChatCompletionsStreamResponse + if err := common.UnmarshalJsonStr(data, &streamResponse); err != nil { + return data + } + injected := false + for i := range streamResponse.Choices { + content := streamResponse.Choices[i].Delta.GetContentString() + if content != "" { + streamResponse.Choices[i].Delta.SetContentString(hint + content) + injected = true + break + } + } + if !injected { + return data + } + info.RouteHintInjected = true + if newData, err := common.Marshal(streamResponse); err == nil { + return string(newData) + } + return data +} + func handleClaudeFormat(c *gin.Context, data string, info *relaycommon.RelayInfo) error { var streamResponse dto.ChatCompletionsStreamResponse if err := common.Unmarshal(common.StringToByteSlice(data), &streamResponse); err != nil { diff --git a/relay/channel/openai/relay-openai.go b/relay/channel/openai/relay-openai.go index de40fe7071fc..9d65d4793932 100644 --- a/relay/channel/openai/relay-openai.go +++ b/relay/channel/openai/relay-openai.go @@ -251,6 +251,16 @@ func OpenaiHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respo applyUsagePostProcessing(info, &simpleResponse.Usage, responseBody) + if info.RelayFormat == types.RelayFormatOpenAI { + if hint := helper.RouteHint(c, info); hint != "" && len(simpleResponse.Choices) > 0 { + message := &simpleResponse.Choices[0].Message + if message.IsStringContent() { + message.SetStringContent(hint + message.StringContent()) + responseBody, _ = common.Marshal(simpleResponse) + } + } + } + switch info.RelayFormat { case types.RelayFormatOpenAI: if usageModified { diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index fa52e05674a2..408343bd0875 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -89,6 +89,7 @@ type RelayInfo struct { TokenId int TokenKey string TokenGroup string + TokenModelRouteNotify bool UserId int UsingGroup string // 使用的分组，当auto跨分组重试时，会变动 UserGroup string // 用户所在分组 @@ -121,6 +122,7 @@ type RelayInfo struct { RelayFormat types.RelayFormat SendResponseCount int ReceivedResponseCount int + RouteHintInjected bool FinalPreConsumedQuota int // 最终预消耗的配额 // ForcePreConsume 为 true 时禁用 BillingSession 的信任额度旁路， // 强制预扣全额。用于异步任务（视频/音乐生成等），因为请求返回后任务仍在运行， @@ -477,6 +479,7 @@ func genBaseRelayInfo(c *gin.Context, request dto.Request) *RelayInfo { TokenKey: common.GetContextKeyString(c, constant.ContextKeyTokenKey), TokenUnlimited: common.GetContextKeyBool(c, constant.ContextKeyTokenUnlimited), TokenGroup: tokenGroup, + TokenModelRouteNotify: common.GetContextKeyBool(c, constant.ContextKeyTokenModelRouteNotify), isFirstResponse: true, RelayMode: relayconstant.Path2RelayMode(c.Request.URL.Path), diff --git a/relay/compatible_handler.go b/relay/compatible_handler.go index a68cfe730f60..30bc7eb8246e 100644 --- a/relay/compatible_handler.go +++ b/relay/compatible_handler.go @@ -193,6 +193,16 @@ func TextHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *types statusCodeMappingStr := c.GetString("status_code_mapping") + if entryModel := common.GetContextKeyString(c, constant.ContextKeyImageAwareEntryModel); entryModel != "" { + reason := "no_image" + if common.GetContextKeyBool(c, constant.ContextKeyImageAwareHasImage) { + reason = "image_detected" + } + c.Header("X-Routed-Model", info.OriginModelName) + c.Header("X-Route-Entry-Model", entryModel) + c.Header("X-Route-Reason", reason) + } + if resp != nil { httpResp = resp.(*http.Response) info.IsStream = info.IsStream || strings.HasPrefix(httpResp.Header.Get("Content-Type"), "text/event-stream") diff --git a/relay/helper/route_hint.go b/relay/helper/route_hint.go new file mode 100644 index 000000000000..83f1e6baa6f4 --- /dev/null +++ b/relay/helper/route_hint.go @@ -0,0 +1,28 @@ +package helper + +import ( + "fmt" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + relaycommon "github.com/QuantumNous/new-api/relay/common" + + "github.com/gin-gonic/gin" +) + +// RouteHint 在 Token 开启 ModelRouteNotify 且本次走了 image-aware 路由时，返回注入响应体的提示文本；否则返回空串。 +func RouteHint(c *gin.Context, info *relaycommon.RelayInfo) string { + if info == nil || !info.TokenModelRouteNotify { + return "" + } + entryModel := common.GetContextKeyString(c, constant.ContextKeyImageAwareEntryModel) + if entryModel == "" { + return "" + } + hasImage := common.GetContextKeyBool(c, constant.ContextKeyImageAwareHasImage) + reason := "no image" + if hasImage { + reason = "image detected" + } + return fmt.Sprintf("> [Route: %s → %s (%s)]\n\n", entryModel, info.OriginModelName, reason) +} diff --git a/service/log_info_generate.go b/service/log_info_generate.go index 54448d59d673..7821ecf79c58 100644 --- a/service/log_info_generate.go +++ b/service/log_info_generate.go @@ -52,6 +52,10 @@ func GenerateTextOtherInfo(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, m other["upstream_model_name"] = relayInfo.UpstreamModelName } + if entryModel := common.GetContextKeyString(ctx, constant.ContextKeyImageAwareEntryModel); entryModel != "" { + other["image_aware_entry_model"] = entryModel + } + isSystemPromptOverwritten := common.GetContextKeyBool(ctx, constant.ContextKeySystemPromptOverride) if isSystemPromptOverwritten { other["is_system_prompt_overwritten"] = true diff --git a/setting/operation_setting/image_aware_routing.go b/setting/operation_setting/image_aware_routing.go new file mode 100644 index 000000000000..89d69ae93a23 --- /dev/null +++ b/setting/operation_setting/image_aware_routing.go @@ -0,0 +1,48 @@ +package operation_setting + +import ( + "sync" + + "github.com/QuantumNous/new-api/common" +) + +// ImageAwareRouteRule 描述一条「虚拟入口模型」的路由规则：含图改写为 VisionModel，否则 CodingModel。 +type ImageAwareRouteRule struct { + VisionModel string `json:"vision_model"` + CodingModel string `json:"coding_model"` +} + +// imageAwareModelRouting 不导出：所有读写必须经由下方加锁方法，避免绕过 imageAwareModelRoutingLock。 +var imageAwareModelRouting = map[string]ImageAwareRouteRule{} + +var imageAwareModelRoutingLock sync.RWMutex + +func ImageAwareModelRouting2JSONString() string { + imageAwareModelRoutingLock.RLock() + defer imageAwareModelRoutingLock.RUnlock() + data, err := common.Marshal(imageAwareModelRouting) + if err != nil { + return "{}" + } + return string(data) +} + +func UpdateImageAwareModelRoutingByJSONString(value string) error { + newMap := make(map[string]ImageAwareRouteRule) + if value != "" { + if err := common.Unmarshal([]byte(value), &newMap); err != nil { + return err + } + } + imageAwareModelRoutingLock.Lock() + imageAwareModelRouting = newMap + imageAwareModelRoutingLock.Unlock() + return nil +} + +func GetImageAwareRouteRule(model string) (ImageAwareRouteRule, bool) { + imageAwareModelRoutingLock.RLock() + defer imageAwareModelRoutingLock.RUnlock() + rule, ok := imageAwareModelRouting[model] + return rule, ok +} diff --git a/web/default/src/features/keys/components/api-keys-mutate-drawer.tsx b/web/default/src/features/keys/components/api-keys-mutate-drawer.tsx index 9a9611baf7c5..bc5ca9165c9f 100644 --- a/web/default/src/features/keys/components/api-keys-mutate-drawer.tsx +++ b/web/default/src/features/keys/components/api-keys-mutate-drawer.tsx @@ -518,6 +518,30 @@ export function ApiKeysMutateDrawer({
+ ( + +
+ + {t('Model Route Notify')} + + + {t( + 'Show a hint in the response when the request is auto-routed to a different model.' + )} + +
+ + + + + )} + /> { @@ -72,6 +73,7 @@ export const API_KEY_FORM_DEFAULT_VALUES: ApiKeyFormValues = { allow_ips: '', group: DEFAULT_GROUP, cross_group_retry: true, + model_route_notify: true, tokenCount: 1, } @@ -109,6 +111,7 @@ export function transformFormDataToPayload( allow_ips: data.allow_ips || '', group: data.group || '', cross_group_retry: data.group === 'auto' ? !!data.cross_group_retry : false, + model_route_notify: !!data.model_route_notify, } } @@ -134,6 +137,7 @@ export function transformApiKeyToFormDefaults( allow_ips: apiKey.allow_ips || '', group: apiKey.group || DEFAULT_GROUP, cross_group_retry: !!apiKey.cross_group_retry, + model_route_notify: apiKey.model_route_notify !== false, tokenCount: 1, } } diff --git a/web/default/src/features/keys/types.ts b/web/default/src/features/keys/types.ts index 1583e6497df7..dbf4588906eb 100644 --- a/web/default/src/features/keys/types.ts +++ b/web/default/src/features/keys/types.ts @@ -42,6 +42,7 @@ export const apiKeySchema = z.object({ }, z.boolean()) .optional() .default(false), + model_route_notify: z.boolean().optional().default(true), model_limits_enabled: z.boolean(), model_limits: z.string().nullish().default(''), allow_ips: z.string().nullish().default(''), @@ -92,6 +93,7 @@ export interface ApiKeyFormData { allow_ips: string group: string cross_group_retry: boolean + model_route_notify: boolean } // ============================================================================ diff --git a/web/default/src/features/system-settings/operations/image-aware-routing-rule-drawer.tsx b/web/default/src/features/system-settings/operations/image-aware-routing-rule-drawer.tsx new file mode 100644 index 000000000000..3e4846a93cf1 --- /dev/null +++ b/web/default/src/features/system-settings/operations/image-aware-routing-rule-drawer.tsx @@ -0,0 +1,232 @@ +/* +Copyright (C) 2023-2026 QuantumNous + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact support@quantumnous.com +*/ +import { useEffect, useMemo } from 'react' +import { useForm } from 'react-hook-form' +import { zodResolver } from '@hookform/resolvers/zod' +import { useQuery } from '@tanstack/react-query' +import * as z from 'zod' +import { useTranslation } from 'react-i18next' +import { Button } from '@/components/ui/button' +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form' +import { Input } from '@/components/ui/input' +import { + Sheet, + SheetClose, + SheetContent, + SheetDescription, + SheetFooter, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet' +import { Combobox } from '@/components/ui/combobox' +import { + SideDrawerSection, + sideDrawerContentClassName, + sideDrawerFooterClassName, + sideDrawerFormClassName, + sideDrawerHeaderClassName, +} from '@/components/drawer-layout' +import { getAllModels } from '@/features/channels/api' + +const ruleSchema = z.object({ + entryModel: z.string().min(1), + visionModel: z.string().min(1), + codingModel: z.string().min(1), +}) + +export type ImageAwareRouteRuleForm = z.infer + +type ImageAwareRoutingRuleDrawerProps = { + open: boolean + onOpenChange: (open: boolean) => void + mode: 'add' | 'edit' + initialValues?: ImageAwareRouteRuleForm | null + onSave: (rule: ImageAwareRouteRuleForm) => Promise | void +} + +export function ImageAwareRoutingRuleDrawer({ + open, + onOpenChange, + mode, + initialValues, + onSave, +}: ImageAwareRoutingRuleDrawerProps) { + const { t } = useTranslation() + const isEdit = mode === 'edit' + + const { data: allModelsData } = useQuery({ + queryKey: ['channel_models'], + queryFn: getAllModels, + }) + + const modelOptions = useMemo( + () => + (allModelsData?.data ?? []) + .map((model) => ({ value: model.id, label: model.id })) + .filter((option) => Boolean(option.value)), + [allModelsData] + ) + + const form = useForm({ + resolver: zodResolver(ruleSchema), + defaultValues: { entryModel: '', visionModel: '', codingModel: '' }, + }) + + useEffect(() => { + if (open) { + form.reset( + initialValues ?? { entryModel: '', visionModel: '', codingModel: '' } + ) + } + }, [open, initialValues, form]) + + const handleSubmit = async (values: ImageAwareRouteRuleForm) => { + const trimmed = { + entryModel: values.entryModel.trim(), + visionModel: values.visionModel.trim(), + codingModel: values.codingModel.trim(), + } + await onSave(trimmed) + onOpenChange(false) + } + + return ( + + + + + {isEdit ? t('Edit Routing Rule') : t('Add Routing Rule')} + + + {t( + 'Map a virtual entry model to a vision model and a coding model based on whether the request contains an image.' + )} + + + +
+ + + ( + + {t('Entry Model')} + + + + + {t( + 'The virtual model name clients send. Not a real model.' + )} + + + + )} + /> + + ( + + {t('Vision Model')} + + + field.onChange(value ?? '') + } + allowCustomValue + placeholder={t('Select vision model')} + searchPlaceholder={t('Search models...')} + emptyText={t('No models found')} + /> + + + {t('Used when the last user message contains an image.')} + + + + )} + /> + + ( + + {t('Coding Model')} + + + field.onChange(value ?? '') + } + allowCustomValue + placeholder={t('Select coding model')} + searchPlaceholder={t('Search models...')} + emptyText={t('No models found')} + /> + + + {t('Used when no image is present in the request.')} + + + + )} + /> + +
+ + + + + } + > + {t('Cancel')} + + + + + + ) +} diff --git a/web/default/src/features/system-settings/operations/image-aware-routing-section.tsx b/web/default/src/features/system-settings/operations/image-aware-routing-section.tsx new file mode 100644 index 000000000000..9707932dac43 --- /dev/null +++ b/web/default/src/features/system-settings/operations/image-aware-routing-section.tsx @@ -0,0 +1,223 @@ +/* +Copyright (C) 2023-2026 QuantumNous + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact support@quantumnous.com +*/ +import { useMemo, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { Camera, Pencil, Plus, Trash2 } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table' +import { SettingsSection } from '../components/settings-section' +import { useUpdateOption } from '../hooks/use-update-option' +import { + ImageAwareRoutingRuleDrawer, + type ImageAwareRouteRuleForm, +} from './image-aware-routing-rule-drawer' + +type ImageAwareRoutingSectionProps = { + defaultValues: { ImageAwareModelRouting: string } +} + +type RouteRule = ImageAwareRouteRuleForm + +function parseRules(json: string): RouteRule[] { + try { + const parsed = JSON.parse(json) + if (parsed && typeof parsed === 'object') { + return Object.entries(parsed).map(([entryModel, value]) => { + const rule = (value ?? {}) as Record + return { + entryModel, + visionModel: String(rule.vision_model ?? ''), + codingModel: String(rule.coding_model ?? ''), + } + }) + } + } catch { + // ignore parse errors, fall back to empty list + } + + return [] +} + +function serializeRules(rules: RouteRule[]): string { + const map: Record = {} + for (const rule of rules) { + if (!rule.entryModel) continue + map[rule.entryModel] = { + vision_model: rule.visionModel, + coding_model: rule.codingModel, + } + } + return JSON.stringify(map) +} + +export function ImageAwareRoutingSection({ + defaultValues, +}: ImageAwareRoutingSectionProps) { + const { t } = useTranslation() + const updateOption = useUpdateOption() + + const rules = useMemo( + () => parseRules(defaultValues.ImageAwareModelRouting), + [defaultValues.ImageAwareModelRouting] + ) + + const [drawerOpen, setDrawerOpen] = useState(false) + const [drawerMode, setDrawerMode] = useState<'add' | 'edit'>('add') + const [editingRule, setEditingRule] = useState(null) + + const persist = async (nextRules: RouteRule[]) => { + await updateOption.mutateAsync({ + key: 'ImageAwareModelRouting', + value: serializeRules(nextRules), + }) + } + + const handleAdd = () => { + setEditingRule(null) + setDrawerMode('add') + setDrawerOpen(true) + } + + const handleEdit = (rule: RouteRule) => { + setEditingRule(rule) + setDrawerMode('edit') + setDrawerOpen(true) + } + + const handleDelete = async (entryModel: string) => { + if (updateOption.isPending) return + await persist(rules.filter((rule) => rule.entryModel !== entryModel)) + } + + const handleSave = async (rule: RouteRule) => { + if (updateOption.isPending) return + if (drawerMode === 'edit' && editingRule) { + const nextRules = rules.map((existing) => + existing.entryModel === editingRule.entryModel ? rule : existing + ) + await persist(nextRules) + } else { + // add：替换同名入口，否则追加 + const exists = rules.some( + (existing) => existing.entryModel === rule.entryModel + ) + const nextRules = exists + ? rules.map((existing) => + existing.entryModel === rule.entryModel ? rule : existing + ) + : [...rules, rule] + await persist(nextRules) + } + } + + return ( + +
+

+ {t( + 'Define virtual entry models that auto-switch between a vision model (when the request contains an image) and a coding model (when it does not). Subsequent text-only turns return to the coding model with prior context.' + )} +

+ + {rules.length === 0 ? ( +
+ {t('No routing rules yet. Click "Add Routing Rule" to create one.')} +
+ ) : ( +
+ + + + {t('Entry Model')} + {t('Vision Model')} + {t('Coding Model')} + + {t('Actions')} + + + + + {rules.map((rule) => ( + + + {rule.entryModel} + + + + + {rule.visionModel || '-'} + + + + {rule.codingModel || '-'} + + +
+ + +
+ + + ))} + +
+
+ )} + +
+ +
+
+ + + + ) +} diff --git a/web/default/src/features/system-settings/operations/index.tsx b/web/default/src/features/system-settings/operations/index.tsx index 6ec679ea1236..105edbcde3cd 100644 --- a/web/default/src/features/system-settings/operations/index.tsx +++ b/web/default/src/features/system-settings/operations/index.tsx @@ -50,6 +50,7 @@ const defaultOperationsSettings: OperationsSettings = { WorkerUrl: '', WorkerValidKey: '', WorkerAllowHttpImageRequestEnabled: false, + ImageAwareModelRouting: '{}', LogConsumeEnabled: false, 'performance_setting.disk_cache_enabled': false, 'performance_setting.disk_cache_threshold_mb': 10, diff --git a/web/default/src/features/system-settings/operations/section-registry.tsx b/web/default/src/features/system-settings/operations/section-registry.tsx index 56861c992773..5c6c9e432bbc 100644 --- a/web/default/src/features/system-settings/operations/section-registry.tsx +++ b/web/default/src/features/system-settings/operations/section-registry.tsx @@ -17,6 +17,7 @@ along with this program. If not, see . For commercial licensing, please contact support@quantumnous.com */ import { SystemBehaviorSection } from '../general/system-behavior-section' +import { ImageAwareRoutingSection } from '../operations/image-aware-routing-section' import { EmailSettingsSection } from '../integrations/email-settings-section' import { MonitoringSettingsSection } from '../integrations/monitoring-settings-section' import { WorkerSettingsSection } from '../integrations/worker-settings-section' @@ -94,6 +95,18 @@ const OPERATIONS_SECTIONS = [ /> ), }, + { + id: 'image-aware-routing', + titleKey: 'Image-Aware Model Routing', + build: (settings: OperationsSettings) => ( + + ), + }, { id: 'logs', titleKey: 'Log Maintenance', diff --git a/web/default/src/features/system-settings/types.ts b/web/default/src/features/system-settings/types.ts index 962bc9ccecc9..5a82566e9adb 100644 --- a/web/default/src/features/system-settings/types.ts +++ b/web/default/src/features/system-settings/types.ts @@ -293,6 +293,7 @@ export type OperationsSettings = { WorkerUrl: string WorkerValidKey: string WorkerAllowHttpImageRequestEnabled: boolean + ImageAwareModelRouting: string LogConsumeEnabled: boolean 'performance_setting.disk_cache_enabled': boolean 'performance_setting.disk_cache_threshold_mb': number diff --git a/web/default/src/features/usage-logs/components/columns/common-logs-columns.tsx b/web/default/src/features/usage-logs/components/columns/common-logs-columns.tsx index ca064fce498a..408171f4fcd6 100644 --- a/web/default/src/features/usage-logs/components/columns/common-logs-columns.tsx +++ b/web/default/src/features/usage-logs/components/columns/common-logs-columns.tsx @@ -533,6 +533,7 @@ export function useCommonLogsColumns(isAdmin: boolean): ColumnDef[] {
) diff --git a/web/default/src/features/usage-logs/components/model-badge.tsx b/web/default/src/features/usage-logs/components/model-badge.tsx index 2c8dedd8593a..cea8da8482ce 100644 --- a/web/default/src/features/usage-logs/components/model-badge.tsx +++ b/web/default/src/features/usage-logs/components/model-badge.tsx @@ -16,7 +16,7 @@ along with this program. If not, see . For commercial licensing, please contact support@quantumnous.com */ -import { Route } from 'lucide-react' +import { Camera, Route } from 'lucide-react' import { useTranslation } from 'react-i18next' import { getLobeIcon } from '@/lib/lobe-icon' import { cn } from '@/lib/utils' @@ -30,6 +30,7 @@ import { StatusBadge } from '@/components/status-badge' interface ModelBadgeProps { modelName: string actualModel?: string + imageAwareEntryModel?: string className?: string } @@ -125,7 +126,9 @@ function ModelBadgeContent(props: ModelBadgeProps) { export function ModelBadge(props: ModelBadgeProps) { const { t } = useTranslation() - if (!props.actualModel) { + const hasRouteInfo = props.actualModel || props.imageAwareEntryModel + + if (!hasRouteInfo) { return } @@ -137,10 +140,25 @@ export function ModelBadge(props: ModelBadgeProps) { } > - + {props.imageAwareEntryModel ? ( + + ) : ( + + )}
+ {props.imageAwareEntryModel && ( +
+ + + {t('Entry Model:')} + + + {props.imageAwareEntryModel} + +
+ )}
{t('Request Model:')} @@ -149,14 +167,16 @@ export function ModelBadge(props: ModelBadgeProps) { {props.modelName}
-
- - {t('Actual Model:')} - - - {props.actualModel} - -
+ {props.actualModel && ( +
+ + {t('Actual Model:')} + + + {props.actualModel} + +
+ )}
diff --git a/web/default/src/features/usage-logs/lib/format.ts b/web/default/src/features/usage-logs/lib/format.ts index 443cada65610..00f69b5b0f1f 100644 --- a/web/default/src/features/usage-logs/lib/format.ts +++ b/web/default/src/features/usage-logs/lib/format.ts @@ -156,6 +156,7 @@ export function formatModelName(log: UsageLog): { name: string isMapped: boolean actualModel?: string + imageAwareEntryModel?: string } { const other = parseLogOther(log.other) const isMapped = !!( @@ -168,6 +169,7 @@ export function formatModelName(log: UsageLog): { name: log.model_name, isMapped, actualModel: isMapped ? other.upstream_model_name : undefined, + imageAwareEntryModel: other?.image_aware_entry_model, } } diff --git a/web/default/src/features/usage-logs/types.ts b/web/default/src/features/usage-logs/types.ts index b243aff3f878..f4de1f473ff5 100644 --- a/web/default/src/features/usage-logs/types.ts +++ b/web/default/src/features/usage-logs/types.ts @@ -154,6 +154,7 @@ export interface LogOtherData { cache_creation_ratio_1h?: number is_model_mapped?: boolean upstream_model_name?: string + image_aware_entry_model?: string audio_ratio?: number audio_completion_ratio?: number frt?: number diff --git a/web/default/src/i18n/locales/en.json b/web/default/src/i18n/locales/en.json index c609827c6d4c..387f2fadf246 100644 --- a/web/default/src/i18n/locales/en.json +++ b/web/default/src/i18n/locales/en.json @@ -1578,6 +1578,7 @@ "Enter your username or email": "Enter your username or email", "Enterprise Account": "Enterprise Account", "Enterprise-grade security with comprehensive permission management": "Enterprise-grade security with comprehensive permission management", + "Entry Model:": "Entry Model:", "Entrypoint (space separated)": "Entrypoint (space separated)", "Env (JSON object)": "Env (JSON object)", "Environment variables": "Environment variables", @@ -2105,6 +2106,20 @@ "Image Preview": "Image Preview", "Image ratio": "Image ratio", "Image to Video": "Image to Video", + "Image-Aware Model Routing": "Image-Aware Model Routing", + "Map a virtual entry model to a vision model and a coding model based on whether the request contains an image.": "Map a virtual entry model to a vision model and a coding model based on whether the request contains an image.", + "Add Routing Rule": "Add Routing Rule", + "Edit Routing Rule": "Edit Routing Rule", + "Entry Model": "Entry Model", + "Vision Model": "Vision Model", + "Coding Model": "Coding Model", + "Select vision model": "Select vision model", + "Select coding model": "Select coding model", + "The virtual model name clients send. Not a real model.": "The virtual model name clients send. Not a real model.", + "Used when the last user message contains an image.": "Used when the last user message contains an image.", + "Used when no image is present in the request.": "Used when no image is present in the request.", + "Define virtual entry models that auto-switch between a vision model (when the request contains an image) and a coding model (when it does not). Subsequent text-only turns return to the coding model with prior context.": "Define virtual entry models that auto-switch between a vision model (when the request contains an image) and a coding model (when it does not). Subsequent text-only turns return to the coding model with prior context.", + "No routing rules yet. Click \"Add Routing Rule\" to create one.": "No routing rules yet. Click \"Add Routing Rule\" to create one.", "Image Tokens": "Image Tokens", "Import to CC Switch": "Import to CC Switch", "In Progress": "In Progress", @@ -2339,6 +2354,7 @@ "Lowest median first-token latency": "Lowest median first-token latency", "m": "m", "Maintenance": "Maintenance", + "Maps a virtual entry model name to a vision model and a coding model. When a request targets an entry model, the gateway inspects the last user message: if it contains an image, the request is routed to the vision model; otherwise to the coding model. Empty ({}) disables the feature. Subsequent text-only turns naturally return to the coding model with prior context.": "Maps a virtual entry model name to a vision model and a coding model. When a request targets an entry model, the gateway inspects the last user message: if it contains an image, the request is routed to the vision model; otherwise to the coding model. Empty ({}) disables the feature. Subsequent text-only turns naturally return to the coding model with prior context.", "Make it easier for teammates to pick the right group.": "Make it easier for teammates to pick the right group.", "Manage": "Manage", "Manage account bindings for this user": "Manage account bindings for this user", @@ -2461,6 +2477,8 @@ "Model mapping must be a JSON object with string values": "Model mapping must be a JSON object with string values", "Model mapping must be valid JSON": "Model mapping must be valid JSON", "Model mapping values must be strings": "Model mapping values must be strings", + "Model Route Notify": "Model Route Notify", + "Show a hint in the response when the request is auto-routed to a different model.": "Show a hint in the response when the request is auto-routed to a different model.", "Model name": "Model name", "Model Name": "Model Name", "Model Name *": "Model Name *", @@ -4556,6 +4574,7 @@ "View details": "View details", "View document": "View document", "View logs": "View logs", + "Virtual Entry Model Routing Map": "Virtual Entry Model Routing Map", "View mode": "View mode", "View model statistics and charts": "View model statistics and charts", "View Pricing": "View Pricing", diff --git a/web/default/src/i18n/locales/zh.json b/web/default/src/i18n/locales/zh.json index 4c718c6c72c6..fc95de664b3b 100644 --- a/web/default/src/i18n/locales/zh.json +++ b/web/default/src/i18n/locales/zh.json @@ -1578,6 +1578,7 @@ "Enter your username or email": "输入您的用户名或电子邮件", "Enterprise Account": "企业账户", "Enterprise-grade security with comprehensive permission management": "企业级安全性，提供全面的权限管理", + "Entry Model:": "入口模型：", "Entrypoint (space separated)": "入口点 (空格分隔)", "Env (JSON object)": "环境变量 (JSON 对象)", "Environment variables": "环境变量", @@ -2105,6 +2106,20 @@ "Image Preview": "图片预览", "Image ratio": "图片倍率", "Image to Video": "图生视频", + "Image-Aware Model Routing": "图片感知模型路由", + "Map a virtual entry model to a vision model and a coding model based on whether the request contains an image.": "将虚拟入口模型根据请求是否包含图片，映射到视觉模型或编程模型。", + "Add Routing Rule": "添加路由规则", + "Edit Routing Rule": "编辑路由规则", + "Entry Model": "入口模型", + "Vision Model": "视觉模型", + "Coding Model": "编程模型", + "Select vision model": "选择视觉模型", + "Select coding model": "选择编程模型", + "The virtual model name clients send. Not a real model.": "客户端发送的虚拟模型名，并非真实模型。", + "Used when the last user message contains an image.": "当最后一条 user 消息包含图片时使用。", + "Used when no image is present in the request.": "当请求中不包含图片时使用。", + "Define virtual entry models that auto-switch between a vision model (when the request contains an image) and a coding model (when it does not). Subsequent text-only turns return to the coding model with prior context.": "定义虚拟入口模型：请求含图片时自动切换到视觉模型，否则使用编程模型。后续纯文本轮次会回到编程模型并保留之前的上下文。", + "No routing rules yet. Click \"Add Routing Rule\" to create one.": "暂无路由规则，点击\"添加路由规则\"创建。", "Image Tokens": "图像 Token", "Import to CC Switch": "填入 CC Switch", "In Progress": "进行中", @@ -2339,6 +2354,7 @@ "Lowest median first-token latency": "最低首 token 延迟中位数", "m": "分钟", "Maintenance": "维护", + "Maps a virtual entry model name to a vision model and a coding model. When a request targets an entry model, the gateway inspects the last user message: if it contains an image, the request is routed to the vision model; otherwise to the coding model. Empty ({}) disables the feature. Subsequent text-only turns naturally return to the coding model with prior context.": "将一个虚拟入口模型名映射到一个视觉模型和一个编程模型。当请求命中入口模型时，网关会检查最后一条 user 消息：若包含图片，则路由到视觉模型；否则路由到编程模型。留空（{}）表示关闭该功能。后续纯文本轮次会自然回到编程模型，并保留之前的上下文。", "Make it easier for teammates to pick the right group.": "让队友更容易选择正确的分组。", "Manage": "管理", "Manage account bindings for this user": "管理此用户的账户绑定", @@ -2461,6 +2477,8 @@ "Model mapping must be a JSON object with string values": "模型映射必须是值为字符串的 JSON 对象", "Model mapping must be valid JSON": "模型映射必须是有效的 JSON", "Model mapping values must be strings": "模型映射的值必须是字符串", + "Model Route Notify": "模型路由提示", + "Show a hint in the response when the request is auto-routed to a different model.": "当请求被自动路由到其他模型时，在响应中显示一条提示。", "Model name": "模型名称", "Model Name": "模型名称", "Model Name *": "模型名称 *", @@ -4556,6 +4574,7 @@ "View details": "查看详情", "View document": "查看文档", "View logs": "查看日志", + "Virtual Entry Model Routing Map": "虚拟入口模型路由表", "View mode": "视图模式", "View model statistics and charts": "查看模型统计和图表", "View Pricing": "查看定价",