diff --git a/.env.example b/.env.example index 3b8a2a5b9534..ae9fac7a838a 100644 --- a/.env.example +++ b/.env.example @@ -118,3 +118,5 @@ LINUX_DO_USER_ENDPOINT=https://connect.linux.do/api/user # 用于验证支付成功/取消回调URL的域名安全性 # 示例: example.com,myapp.io 将允许 example.com, sub.example.com, myapp.io 等 # TRUSTED_REDIRECT_DOMAINS=example.com,myapp.io +# Intelligent routing is configured through versioned system options under +# intelligent_routing_setting.* and starts disabled in shadow-only mode. diff --git a/.gitignore b/.gitignore index dc328dd6c80c..7a169e2b517f 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ electron/dist token_estimator_test.go skills-lock.json .playwright-mcp +.worktrees/ # Local-only live probes and scratch test workspaces. .local-tests/ diff --git a/controller/audit.go b/controller/audit.go index d6974b900806..fa9f0e20c89b 100644 --- a/controller/audit.go +++ b/controller/audit.go @@ -16,19 +16,24 @@ import ( // action 的 params 填充。本地化展示文案在前端 i18n 模板中维护，本表是语言中立的 // 英文基线——调用方因此无需在每个埋点处手写句子（避免与 params 重复书写同一份值）。 var auditContentTemplates = map[string]string{ - "user.create": "Created user ${username} (role ${role})", - "user.update": "Updated user ${username} (ID: ${id})", - "user.delete": "Deleted user ${username} (ID: ${id})", - "user.manage": "Performed ${action} on user ${username} (ID: ${id})", - "user.quota_add": "Increased user quota by ${quota}", - "user.quota_subtract": "Decreased user quota by ${quota}", - "user.quota_override": "Overrode user quota from ${from} to ${to}", - "user.binding_clear": "Cleared ${bindingType} binding for user ${username}", - "user.2fa_disable": "Force-disabled two-factor authentication for the user", - "user.passkey_register": "Registered a passkey", - "user.passkey_delete": "Deleted a passkey", - "user.reset_passkey": "Reset the user passkey", - "option.update": "Updated system setting ${key}", + "user.create": "Created user ${username} (role ${role})", + "user.update": "Updated user ${username} (ID: ${id})", + "user.delete": "Deleted user ${username} (ID: ${id})", + "user.manage": "Performed ${action} on user ${username} (ID: ${id})", + "user.quota_add": "Increased user quota by ${quota}", + "user.quota_subtract": "Decreased user quota by ${quota}", + "user.quota_override": "Overrode user quota from ${from} to ${to}", + "user.binding_clear": "Cleared ${bindingType} binding for user ${username}", + "user.2fa_disable": "Force-disabled two-factor authentication for the user", + "user.passkey_register": "Registered a passkey", + "user.passkey_delete": "Deleted a passkey", + "user.reset_passkey": "Reset the user passkey", + "option.update": "Updated system setting ${key}", + "intelligent_routing.policy.create": "Created intelligent routing policy ${id}", + "intelligent_routing.policy.update": "Updated intelligent routing policy ${id}", + "intelligent_routing.policy.publish": "Published intelligent routing policy ${version}", + "intelligent_routing.policy.rollback": "Rolled back intelligent routing policy to ${source_version} as ${version}", + "intelligent_routing.rollout.update": "Updated intelligent routing rollout revision ${revision}", "channel.create": "Created channel ${name} (type ${type}, count ${count})", "channel.update": "Updated channel ${name} (ID: ${id})", diff --git a/controller/intelligent_routing.go b/controller/intelligent_routing.go new file mode 100644 index 000000000000..356ae176a578 --- /dev/null +++ b/controller/intelligent_routing.go @@ -0,0 +1,207 @@ +package controller + +import ( + "errors" + "net/http" + "strconv" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/model" + intelligentrouting "github.com/QuantumNous/new-api/service/intelligent_routing" + "github.com/gin-gonic/gin" +) + +func ListIntelligentRoutingPolicies(c *gin.Context) { + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20")) + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = 20 + } + if pageSize > 100 { + pageSize = 100 + } + policies, total, err := model.ListIntelligentRoutingPolicies((page-1)*pageSize, pageSize) + if err != nil { + intelligentRoutingError(c, err) + return + } + c.JSON(http.StatusOK, gin.H{"success": true, "data": policies, "total": total, "page": page, "page_size": pageSize}) +} + +func GetIntelligentRoutingPolicy(c *gin.Context) { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id < 1 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy id"}) + return + } + policy, err := model.GetIntelligentRoutingPolicy(id) + if err != nil { + intelligentRoutingError(c, err) + return + } + c.JSON(http.StatusOK, gin.H{"success": true, "data": policy}) +} + +func CreateIntelligentRoutingPolicy(c *gin.Context) { + var request dto.IntelligentRoutingDraftRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) + return + } + policy, issues, err := intelligentrouting.DefaultPolicyControl.CreateDraft(c, request.Config, c.GetInt("id")) + if err != nil { + intelligentRoutingError(c, err) + return + } + if len(issues) > 0 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) + return + } + recordManageAudit(c, "intelligent_routing.policy.create", map[string]interface{}{"id": policy.Id}) + c.JSON(http.StatusCreated, gin.H{"success": true, "data": policy}) +} + +func UpdateIntelligentRoutingPolicy(c *gin.Context) { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id < 1 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy id"}) + return + } + var request dto.IntelligentRoutingDraftUpdateRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) + return + } + policy, issues, err := intelligentrouting.DefaultPolicyControl.UpdateDraft(c, id, request.UpdatedAt, request.Config) + if err != nil { + intelligentRoutingError(c, err) + return + } + if len(issues) > 0 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) + return + } + recordManageAudit(c, "intelligent_routing.policy.update", map[string]interface{}{"id": policy.Id}) + c.JSON(http.StatusOK, gin.H{"success": true, "data": policy}) +} + +func ValidateIntelligentRoutingPolicy(c *gin.Context) { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id < 1 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy id"}) + return + } + policy, err := model.GetIntelligentRoutingPolicy(id) + if err != nil { + intelligentRoutingError(c, err) + return + } + validated, issues := intelligentrouting.ValidatePolicyDocument(policy.Config) + c.JSON(http.StatusOK, gin.H{"success": len(issues) == 0, "data": gin.H{"checksum": validated.Checksum, "issues": issues}}) +} + +func PublishIntelligentRoutingPolicy(c *gin.Context) { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id < 1 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy id"}) + return + } + var request dto.IntelligentRoutingPublishRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) + return + } + policy, issues, err := intelligentrouting.DefaultPolicyControl.Publish(c, id, c.GetInt("id"), request.ChangeNote) + if err != nil { + intelligentRoutingError(c, err) + return + } + if len(issues) > 0 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) + return + } + recordManageAudit(c, "intelligent_routing.policy.publish", map[string]interface{}{"id": policy.Id, "version": policy.Version}) + c.JSON(http.StatusOK, gin.H{"success": true, "data": policy}) +} + +func RollbackIntelligentRoutingPolicy(c *gin.Context) { + version, err := strconv.Atoi(c.Param("version")) + if err != nil || version < 1 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy version"}) + return + } + var request dto.IntelligentRoutingPublishRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) + return + } + policy, issues, err := intelligentrouting.DefaultPolicyControl.Rollback(c, version, c.GetInt("id"), request.ChangeNote) + if err != nil { + intelligentRoutingError(c, err) + return + } + if len(issues) > 0 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) + return + } + recordManageAudit(c, "intelligent_routing.policy.rollback", map[string]interface{}{"source_version": version, "version": policy.Version}) + c.JSON(http.StatusOK, gin.H{"success": true, "data": policy}) +} + +func GetIntelligentRoutingRollout(c *gin.Context) { + rollout, err := model.GetIntelligentRoutingRollout() + if err != nil { + intelligentRoutingError(c, err) + return + } + c.JSON(http.StatusOK, gin.H{"success": true, "data": rollout}) +} + +func UpdateIntelligentRoutingRollout(c *gin.Context) { + var request dto.IntelligentRoutingRolloutUpdateRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) + return + } + userGroups, err := common.Marshal(request.UserGroups) + if err != nil { + intelligentRoutingError(c, err) + return + } + tokenGroups, err := common.Marshal(request.TokenGroups) + if err != nil { + intelligentRoutingError(c, err) + return + } + rollout, issues, err := intelligentrouting.DefaultPolicyControl.UpdateRollout(c, request.Revision, model.IntelligentRoutingRollout{ + PolicyVersion: request.PolicyVersion, Enabled: request.Enabled, Mode: request.Mode, TrafficPercent: request.TrafficPercent, + UserGroups: string(userGroups), TokenGroups: string(tokenGroups), UpdatedBy: c.GetInt("id"), + }) + if err != nil { + intelligentRoutingError(c, err) + return + } + if len(issues) > 0 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) + return + } + recordManageAudit(c, "intelligent_routing.rollout.update", map[string]interface{}{"revision": rollout.Revision, "policy_version": rollout.PolicyVersion, "mode": rollout.Mode, "traffic_percent": rollout.TrafficPercent}) + c.JSON(http.StatusOK, gin.H{"success": true, "data": rollout}) +} + +func intelligentRoutingError(c *gin.Context, err error) { + switch { + case errors.Is(err, model.ErrIntelligentRoutingPolicyNotFound), errors.Is(err, model.ErrIntelligentRoutingRolloutNotFound): + c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "resource not found"}) + case errors.Is(err, model.ErrIntelligentRoutingRevisionConflict): + c.JSON(http.StatusConflict, gin.H{"success": false, "message": "revision conflict"}) + case errors.Is(err, model.ErrIntelligentRoutingPolicyImmutable): + c.JSON(http.StatusConflict, gin.H{"success": false, "message": "published policy is immutable"}) + default: + c.JSON(http.StatusServiceUnavailable, gin.H{"success": false, "message": "intelligent routing service unavailable"}) + } +} diff --git a/controller/intelligent_routing_shadow_test.go b/controller/intelligent_routing_shadow_test.go new file mode 100644 index 000000000000..858e141bc811 --- /dev/null +++ b/controller/intelligent_routing_shadow_test.go @@ -0,0 +1,161 @@ +package controller + +import ( + "net/http/httptest" + "testing" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + relayconstant "github.com/QuantumNous/new-api/relay/constant" + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/types" + intelligentrouting "github.com/QuantumNous/new-api/service/intelligent_routing" + routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" + "github.com/QuantumNous/new-api/setting/ratio_setting" + hosttypes "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildShadowRoutePlanDoesNotChangeLiveModelOrChannel(t *testing.T) { + saved := routingsetting.Get() + t.Cleanup(func() { require.NoError(t, routingsetting.Update(saved)) }) + require.NoError(t, routingsetting.Update(routingsetting.Config{ + Enabled: true, ShadowOnly: true, + Models: []routingsetting.ModelPolicy{{Model: "cheap", Tier: 1, InputPrice: 1, OutputPrice: 2, ContextLimit: 8192}}, + })) + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = httptest.NewRequest("POST", "/v1/chat/completions", nil) + common.SetContextKey(ctx, constant.ContextKeyChannelId, 99) + request := &dto.GeneralOpenAIRequest{Model: "client-model", Messages: []dto.Message{{Role: "user", Content: "hello"}}} + info := &relaycommon.RelayInfo{OriginModelName: "client-model", TokenGroup: "default", Request: request, RelayFormat: types.RelayFormatOpenAI} + err := buildShadowRoutePlan(ctx, info, 120, func(string, string) []*model.Channel { + return []*model.Channel{{Id: 7, Status: common.ChannelStatusEnabled, Models: "cheap", Group: "default"}} + }) + require.NoError(t, err) + require.NotNil(t, info.IntelligentRoutePlan) + assert.Equal(t, "client-model", info.OriginModelName) + assert.Equal(t, 99, common.GetContextKeyInt(ctx, constant.ContextKeyChannelId)) + assert.Equal(t, "cheap", info.IntelligentRoutePlan.Nodes[0].Model) +} + +func TestSupportsIntelligentRoutingOnlyForOpenAITextEndpoints(t *testing.T) { + tests := []struct { + name string + format types.RelayFormat + mode int + want bool + }{ + {name: "chat", format: types.RelayFormatOpenAI, mode: relayconstant.RelayModeChatCompletions, want: true}, + {name: "responses", format: types.RelayFormatOpenAIResponses, mode: relayconstant.RelayModeResponses, want: true}, + {name: "responses compact", format: types.RelayFormatOpenAIResponsesCompaction, mode: relayconstant.RelayModeResponsesCompact, want: true}, + {name: "images", format: types.RelayFormatOpenAI, mode: relayconstant.RelayModeImagesGenerations}, + {name: "claude", format: types.RelayFormatClaude, mode: relayconstant.RelayModeChatCompletions}, + {name: "realtime", format: types.RelayFormatOpenAIRealtime, mode: relayconstant.RelayModeChatCompletions}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.want, supportsIntelligentRouting(test.format, test.mode)) + }) + } +} + +func TestComputeLiveRoutePricingPreconsumesMostExpensiveCandidateAndRestoresFirst(t *testing.T) { + savedPrices := ratio_setting.ModelPrice2JSONString() + t.Cleanup(func() { require.NoError(t, ratio_setting.UpdateModelPriceByJSONString(savedPrices)) }) + prices, err := common.Marshal(map[string]float64{"cheap": 0.1, "fallback": 0.4}) + require.NoError(t, err) + require.NoError(t, ratio_setting.UpdateModelPriceByJSONString(string(prices))) + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + info := &relaycommon.RelayInfo{OriginModelName: "requested", UserGroup: "default", UsingGroup: "default", Request: &dto.GeneralOpenAIRequest{Model: "requested"}} + plan := &hosttypes.IntelligentRoutePlan{Nodes: []hosttypes.IntelligentRouteNode{{Model: "cheap"}, {Model: "fallback"}}} + priceData, err := computeLiveRoutePricing(ctx, info, plan, 100, &types.TokenCountMeta{}) + require.NoError(t, err) + assert.Equal(t, "cheap", info.GetExecutionModelName()) + assert.Equal(t, 200000, priceData.QuotaToPreConsume) + assert.Equal(t, 0.1, priceData.ModelPrice) +} + +func TestApplyIntelligentRouteNodeSwitchesExecutionWithoutChangingRequestedModel(t *testing.T) { + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = httptest.NewRequest("POST", "/v1/chat/completions", nil) + request := &dto.GeneralOpenAIRequest{Model: "requested"} + info := &relaycommon.RelayInfo{OriginModelName: "requested", Request: request} + channel := &model.Channel{Id: 7, Status: common.ChannelStatusEnabled, Models: "cheap", Group: "default", Key: "test-key"} + err := applyIntelligentRouteNode(ctx, info, channel, hosttypes.IntelligentRouteNode{Model: "cheap", ChannelID: 7}) + require.Nil(t, err) + assert.Equal(t, "requested", info.OriginModelName) + assert.Equal(t, "cheap", info.GetExecutionModelName()) + assert.Equal(t, "cheap", request.Model) + assert.Equal(t, 7, info.ChannelId) +} + +func TestRecordIntelligentRouteHealthTracksOnlyLiveSelectedChannel(t *testing.T) { + var tracker intelligentrouting.HealthTracker + info := &relaycommon.RelayInfo{IntelligentRoutePlan: &hosttypes.IntelligentRoutePlan{}, IntelligentRouteShadow: false, ChannelMeta: &relaycommon.ChannelMeta{ChannelId: 7}} + recordIntelligentRouteHealth(&tracker, info, false) + for i := 1; i < 20; i++ { + tracker.Record(7, false) + } + assert.Equal(t, intelligentrouting.HealthOpen, tracker.Snapshot(7).Tier) + + shadow := &relaycommon.RelayInfo{IntelligentRoutePlan: &hosttypes.IntelligentRoutePlan{}, IntelligentRouteShadow: true, ChannelMeta: &relaycommon.ChannelMeta{ChannelId: 8}} + recordIntelligentRouteHealth(&tracker, shadow, false) + assert.Equal(t, intelligentrouting.HealthProbation, tracker.Snapshot(8).Tier) +} + +func TestBuildRoutePlanPrefersAffordableStickyNode(t *testing.T) { + saved := routingsetting.Get() + t.Cleanup(func() { require.NoError(t, routingsetting.Update(saved)) }) + require.NoError(t, routingsetting.Update(routingsetting.Config{ + Enabled: true, MaxAttempts: 3, + Models: []routingsetting.ModelPolicy{ + {Model: "cheapest", Tier: 1, InputPrice: 1, OutputPrice: 1}, + {Model: "sticky", Tier: 1, InputPrice: 1.1, OutputPrice: 1.1}, + {Model: "safest", Tier: 1, InputPrice: 5, OutputPrice: 5}, + }, + })) + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = httptest.NewRequest("POST", "/v1/chat/completions", nil) + ctx.Request.Header.Set("X-Session-ID", "sticky-test-session") + request := &dto.GeneralOpenAIRequest{Model: "requested", Messages: []dto.Message{{Role: "user", Content: "hello"}}} + info := &relaycommon.RelayInfo{UserId: 42, OriginModelName: "requested", TokenGroup: "default", Request: request, RelayFormat: types.RelayFormatOpenAI} + key := intelligentrouting.ConversationKey("42", "sticky-test-session", "hello") + intelligentrouting.DefaultStickinessStore.Record(key, intelligentrouting.TaskGeneral, intelligentrouting.StickyRoute{Model: "sticky", ChannelID: 2}) + err := buildShadowRoutePlan(ctx, info, 100, func(string, string) []*model.Channel { + return []*model.Channel{ + {Id: 1, Status: common.ChannelStatusEnabled, Models: "cheapest", Group: "default"}, + {Id: 2, Status: common.ChannelStatusEnabled, Models: "sticky", Group: "default"}, + {Id: 3, Status: common.ChannelStatusEnabled, Models: "safest", Group: "default"}, + } + }) + require.NoError(t, err) + assert.Equal(t, "sticky", info.IntelligentRoutePlan.Nodes[0].Model) + assert.Equal(t, key, info.IntelligentRouteSessionKey) +} + +func TestRecordIntelligentRouteSuccessCreatesStickyRoute(t *testing.T) { + var store intelligentrouting.StickinessStore + info := &relaycommon.RelayInfo{ + ExecutionModelName: "cheap", IntelligentRouteSessionKey: "session-key", IntelligentRouteTask: string(intelligentrouting.TaskSummary), + IntelligentRoutePlan: &hosttypes.IntelligentRoutePlan{}, ChannelMeta: &relaycommon.ChannelMeta{ChannelId: 7}, + } + recordIntelligentRouteSuccess(&store, info) + route, ok := store.Get("session-key", intelligentrouting.TaskSummary) + require.True(t, ok) + assert.Equal(t, "cheap", route.Model) + assert.Equal(t, 7, route.ChannelID) +} + +func TestRecordIntelligentRouteAttemptCapturesOutcomeAndLatency(t *testing.T) { + info := &relaycommon.RelayInfo{IntelligentRoutePlan: &hosttypes.IntelligentRoutePlan{}} + node := hosttypes.IntelligentRouteNode{Model: "cheap", ChannelID: 7} + recordIntelligentRouteAttempt(info, node, 1, time.UnixMilli(1000), time.UnixMilli(1125), "failed", "timeout") + require.Len(t, info.IntelligentRouteAttempts, 1) + assert.Equal(t, int64(125), info.IntelligentRouteAttempts[0].LatencyMS) + assert.Equal(t, "timeout", info.IntelligentRouteAttempts[0].FailureReason) +} diff --git a/controller/relay.go b/controller/relay.go index 8dccfe76dddd..53be1f2b14d6 100644 --- a/controller/relay.go +++ b/controller/relay.go @@ -6,6 +6,7 @@ import ( "io" "log" "net/http" + "strconv" "strings" "time" @@ -23,8 +24,11 @@ import ( "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/service" + intelligentrouting "github.com/QuantumNous/new-api/service/intelligent_routing" "github.com/QuantumNous/new-api/setting" + routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" "github.com/QuantumNous/new-api/setting/operation_setting" + hosttypes "github.com/QuantumNous/new-api/types" "github.com/bytedance/gopkg/util/gopool" "github.com/samber/lo" @@ -152,8 +156,45 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { } relayInfo.SetEstimatePromptTokens(tokens) + routingConfig := routingsetting.Get() + runtimeSnapshot := intelligentrouting.DefaultPolicyControl.Snapshot() + if runtimeSnapshot.Rollout.Exists { + decision := intelligentrouting.ResolveRollout(runtimeSnapshot, intelligentrouting.RolloutSubject{ + AccountID: relayInfo.UserId, + TokenID: relayInfo.TokenId, + UserGroup: relayInfo.UserGroup, + TokenGroup: relayInfo.TokenGroup, + }) + if decision.Mode == model.IntelligentRoutingModeLive && !intelligentrouting.DefaultSharedRuntime.Ready() { + decision.Selected = false + } + routingConfig = runtimeSnapshot.Config + routingConfig.Enabled = decision.Selected + routingConfig.ShadowOnly = decision.Mode == model.IntelligentRoutingModeShadow + relayInfo.IntelligentRoutePolicyVersion = decision.PolicyVersion + relayInfo.IntelligentRouteRolloutRevision = decision.Revision + relayInfo.IntelligentRouteRolloutBucket = decision.Bucket + relayInfo.IntelligentRouteRolloutMode = decision.Mode + } + intelligentRoutingActive := routingConfig.Enabled && supportsIntelligentRouting(relayInfo.RelayFormat, relayInfo.RelayMode) + if intelligentRoutingActive { + if routingErr := buildRoutePlan(c, relayInfo, tokens, nil, routingConfig); routingErr != nil { + relayInfo.IntelligentRouteError = routingErr.Error() + if routingConfig.ShadowOnly { + logger.LogWarn(c, "intelligent routing shadow plan failed: "+routingErr.Error()) + } else { + newAPIError = types.NewError(routingErr, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) + return + } + } + } - priceData, err := helper.ModelPriceHelper(c, relayInfo, tokens, meta) + var priceData hosttypes.PriceData + if intelligentRoutingActive && !routingConfig.ShadowOnly { + priceData, err = computeLiveRoutePricing(c, relayInfo, relayInfo.IntelligentRoutePlan, tokens, meta) + } else { + priceData, err = helper.ModelPriceHelper(c, relayInfo, tokens, meta) + } if err != nil { newAPIError = types.NewError(err, types.ErrorCodeModelPriceError, types.ErrOptionWithStatusCode(http.StatusBadRequest)) return @@ -191,17 +232,60 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { relayInfo.RetryIndex = 0 relayInfo.LastError = nil - for ; retryParam.GetRetry() <= common.RetryTimes; retryParam.IncreaseRetry() { + maxRetries := common.RetryTimes + var routeBudget *intelligentrouting.ExecutionBudget + if intelligentRoutingActive && !routingConfig.ShadowOnly && relayInfo.IntelligentRoutePlan != nil { + maxRetries = min(routingConfig.MaxAttempts, len(relayInfo.IntelligentRoutePlan.Nodes)) - 1 + duration := routingConfig.NonStreamBudget + if relayInfo.IsStream { + duration = routingConfig.StreamFirstByteBudget + } + routeBudget = intelligentrouting.NewExecutionBudget(relayInfo.IntelligentRoutePlan.Nodes, routingConfig.MaxCostMultiplier, duration, time.Now()) + } + for ; retryParam.GetRetry() <= maxRetries; retryParam.IncreaseRetry() { + var budgetNode *hosttypes.IntelligentRouteNode + attemptStarted := time.Now() + if routeBudget != nil { + index, allowed := routeBudget.SelectAttempt(relayInfo.IntelligentRoutePlan.Nodes, retryParam.GetRetry(), time.Now()) + if !allowed { + break + } + retryParam.SetRetry(index) + budgetNode = &relayInfo.IntelligentRoutePlan.Nodes[index] + } relayInfo.RetryIndex = retryParam.GetRetry() channel, channelErr := getChannel(c, relayInfo, retryParam) if channelErr != nil { + if budgetNode != nil { + recordIntelligentRouteAttempt(relayInfo, *budgetNode, retryParam.GetRetry(), attemptStarted, time.Now(), "rejected", channelErr.Error()) + } logger.LogError(c, channelErr.Error()) newAPIError = channelErr + if intelligentRoutingActive && !routingConfig.ShadowOnly && retryParam.GetRetry() < maxRetries { + relayInfo.LastError = channelErr + continue + } break } addUsedChannel(c, channel.Id) + if intelligentRoutingActive && !routingConfig.ShadowOnly { + if _, pricingErr := helper.ModelPriceHelper(c, relayInfo, tokens, meta); pricingErr != nil { + newAPIError = types.NewError(pricingErr, types.ErrorCodeModelPriceError, types.ErrOptionWithSkipRetry()) + if budgetNode != nil { + recordIntelligentRouteAttempt(relayInfo, *budgetNode, retryParam.GetRetry(), attemptStarted, time.Now(), "rejected", newAPIError.Error()) + } + if retryParam.GetRetry() < maxRetries { + relayInfo.LastError = newAPIError + continue + } + break + } + } if billingErr := service.PrepareTieredBillingForSelectedGroup(c, relayInfo); billingErr != nil { newAPIError = billingErr + if budgetNode != nil { + recordIntelligentRouteAttempt(relayInfo, *budgetNode, retryParam.GetRetry(), attemptStarted, time.Now(), "rejected", billingErr.Error()) + } break } @@ -213,9 +297,15 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { } else { newAPIError = types.NewErrorWithStatusCode(bodyErr, types.ErrorCodeReadRequestBodyFailed, http.StatusBadRequest, types.ErrOptionWithSkipRetry()) } + if budgetNode != nil { + recordIntelligentRouteAttempt(relayInfo, *budgetNode, retryParam.GetRetry(), attemptStarted, time.Now(), "rejected", newAPIError.Error()) + } break } c.Request.Body = io.NopCloser(bodyStorage) + if budgetNode != nil { + routeBudget.Record(*budgetNode) + } switch relayFormat { case types.RelayFormatOpenAIRealtime: @@ -229,16 +319,30 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { } if newAPIError == nil { + if budgetNode != nil { + recordIntelligentRouteAttempt(relayInfo, *budgetNode, retryParam.GetRetry(), attemptStarted, time.Now(), "success", "") + } + recordIntelligentRouteHealth(&intelligentrouting.DefaultHealthTracker, relayInfo, true) + recordIntelligentRouteSuccess(&intelligentrouting.DefaultStickinessStore, relayInfo) + intelligentrouting.DefaultQualityTracker.Record(relayInfo.GetExecutionModelName(), intelligentrouting.TaskType(relayInfo.IntelligentRouteTask), true) relayInfo.LastError = nil return } + if budgetNode != nil { + recordIntelligentRouteAttempt(relayInfo, *budgetNode, retryParam.GetRetry(), attemptStarted, time.Now(), "failed", newAPIError.Error()) + } + if newAPIError.GetErrorCode() == types.ErrorCodeBadResponseBody { + intelligentrouting.DefaultStickinessStore.RecordValidationFailure(relayInfo.IntelligentRouteSessionKey) + intelligentrouting.DefaultQualityTracker.Record(relayInfo.GetExecutionModelName(), intelligentrouting.TaskType(relayInfo.IntelligentRouteTask), false) + } + recordIntelligentRouteHealth(&intelligentrouting.DefaultHealthTracker, relayInfo, false) newAPIError = service.NormalizeViolationFeeError(newAPIError) relayInfo.LastError = newAPIError processChannelError(c, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(c, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError) - if !shouldRetry(c, newAPIError, common.RetryTimes-retryParam.GetRetry()) { + if !shouldRetry(c, newAPIError, maxRetries-retryParam.GetRetry()) { break } } @@ -255,6 +359,113 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { } } +func supportsIntelligentRouting(relayFormat types.RelayFormat, relayMode int) bool { + switch relayFormat { + case types.RelayFormatOpenAI: + return relayMode == relayconstant.RelayModeChatCompletions + case types.RelayFormatOpenAIResponses: + return relayMode == relayconstant.RelayModeResponses + case types.RelayFormatOpenAIResponsesCompaction: + return relayMode == relayconstant.RelayModeResponsesCompact + default: + return false + } +} + +func recordIntelligentRouteHealth(tracker *intelligentrouting.HealthTracker, info *relaycommon.RelayInfo, success bool) { + if tracker == nil || info == nil || info.IntelligentRoutePlan == nil || info.IntelligentRouteShadow || info.GetChannelID() == 0 { + return + } + tracker.Record(info.GetChannelID(), success) +} + +func recordIntelligentRouteSuccess(store *intelligentrouting.StickinessStore, info *relaycommon.RelayInfo) { + if store == nil || info == nil || info.IntelligentRoutePlan == nil || info.IntelligentRouteShadow || info.IntelligentRouteSessionKey == "" || info.GetChannelID() == 0 { + return + } + store.Record(info.IntelligentRouteSessionKey, intelligentrouting.TaskType(info.IntelligentRouteTask), intelligentrouting.StickyRoute{ + Model: info.GetExecutionModelName(), ChannelID: info.GetChannelID(), + }) +} + +func recordIntelligentRouteAttempt(info *relaycommon.RelayInfo, node hosttypes.IntelligentRouteNode, index int, startedAt, finishedAt time.Time, outcome, failureReason string) { + if info == nil || info.IntelligentRoutePlan == nil { + return + } + if len(failureReason) > 512 { + failureReason = failureReason[:512] + } + latency := finishedAt.Sub(startedAt).Milliseconds() + if latency < 0 { + latency = 0 + } + info.IntelligentRouteAttempts = append(info.IntelligentRouteAttempts, hosttypes.IntelligentRouteAttempt{ + Index: index, Model: node.Model, ChannelID: node.ChannelID, Outcome: outcome, FailureReason: failureReason, LatencyMS: latency, + }) +} + +func buildShadowRoutePlan(c *gin.Context, info *relaycommon.RelayInfo, promptTokens int, source intelligentrouting.ChannelSource) error { + return buildRoutePlan(c, info, promptTokens, source, routingsetting.Get()) +} + +func buildRoutePlan(c *gin.Context, info *relaycommon.RelayInfo, promptTokens int, source intelligentrouting.ChannelSource, config routingsetting.Config) error { + startedAt := time.Now() + features := intelligentrouting.ExtractFeatures(intelligentrouting.Input{ + Request: info.Request, RelayFormat: info.RelayFormat, PromptTokens: promptTokens, RequestPath: c.Request.URL.Path, + }) + requirements := intelligentrouting.DeriveRequirements(features) + candidates := intelligentrouting.NewCatalog(config, source).Build(info.TokenGroup, c.Request.URL.Path) + for i := range candidates { + candidates[i].PredictedSuccess = intelligentrouting.DefaultQualityTracker.Predict(candidates[i].Model, features.Task, candidates[i].PredictedSuccess) + } + sessionKey := intelligentrouting.ConversationKey(strconv.Itoa(info.UserId), c.GetHeader("X-Session-ID"), intelligentrouting.ConversationSeed(info.Request)) + preferred, _ := intelligentrouting.DefaultStickinessStore.Get(sessionKey, features.Task) + plan, err := intelligentrouting.Plan(intelligentrouting.PlanInput{ + RequestedModel: info.OriginModelName, PolicyVersion: config.PolicyVersion, + Features: features, Requirements: requirements, Candidates: candidates, + QualityThreshold: config.QualityThresholds[features.Task], + MaxAttempts: config.MaxAttempts, MaxEndpointsPerModel: config.MaxEndpointsPerModel, + MaxCostMultiplier: config.MaxCostMultiplier, + PreferredModel: preferred.Model, PreferredChannelID: preferred.ChannelID, + }) + if err != nil { + intelligentrouting.DefaultMetrics.Observe(intelligentrouting.Observation{NoRoute: true, PlanningDuration: time.Since(startedAt)}) + return err + } + info.IntelligentRoutePlan = &plan + info.IntelligentRouteShadow = config.ShadowOnly + info.IntelligentRouteLive = !config.ShadowOnly + info.IntelligentRouteSessionKey = sessionKey + info.IntelligentRouteTask = string(features.Task) + intelligentrouting.DefaultMetrics.Observe(intelligentrouting.Observation{CandidateTier: plan.Nodes[0].Tier, PlanningDuration: time.Since(startedAt)}) + return nil +} + +func computeLiveRoutePricing(c *gin.Context, info *relaycommon.RelayInfo, plan *hosttypes.IntelligentRoutePlan, promptTokens int, meta *types.TokenCountMeta) (hosttypes.PriceData, error) { + if plan == nil || len(plan.Nodes) == 0 { + return hosttypes.PriceData{}, errors.New("live route plan is empty") + } + maxPreConsume := 0 + for _, node := range plan.Nodes { + info.SetExecutionModelName(node.Model) + priceData, err := helper.ModelPriceHelper(c, info, promptTokens, meta) + if err != nil { + return hosttypes.PriceData{}, fmt.Errorf("price live route model %s: %w", node.Model, err) + } + if priceData.QuotaToPreConsume > maxPreConsume { + maxPreConsume = priceData.QuotaToPreConsume + } + } + info.SetExecutionModelName(plan.Nodes[0].Model) + firstPrice, err := helper.ModelPriceHelper(c, info, promptTokens, meta) + if err != nil { + return hosttypes.PriceData{}, err + } + firstPrice.QuotaToPreConsume = maxPreConsume + info.PriceData = firstPrice + return firstPrice, nil +} + var upgrader = websocket.Upgrader{ Subprotocols: []string{"realtime"}, // WS 握手支持的协议，如果有使用 Sec-WebSocket-Protocol，则必须在此声明对应的 Protocol TODO add other protocol CheckOrigin: func(r *http.Request) bool { @@ -298,6 +509,22 @@ func fastTokenCountMetaForPricing(request dto.Request) *types.TokenCountMeta { } func getChannel(c *gin.Context, info *relaycommon.RelayInfo, retryParam *service.RetryParam) (*model.Channel, *types.NewAPIError) { + if info.IntelligentRouteLive && info.IntelligentRoutePlan != nil { + index := retryParam.GetRetry() + if index < 0 || index >= len(info.IntelligentRoutePlan.Nodes) { + return nil, types.NewError(errors.New("intelligent route attempts exhausted"), types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) + } + node := info.IntelligentRoutePlan.Nodes[index] + channel, err := model.CacheGetChannel(node.ChannelID) + if err != nil || channel == nil { + return nil, types.NewError(fmt.Errorf("intelligent route channel %d unavailable: %v", node.ChannelID, err), types.ErrorCodeGetChannelFailed) + } + if setupErr := applyIntelligentRouteNode(c, info, channel, node); setupErr != nil { + return nil, setupErr + } + info.IntelligentRouteAttempt = index + return channel, nil + } if info.ChannelMeta == nil { autoBan := c.GetBool("auto_ban") autoBanInt := 1 @@ -328,6 +555,15 @@ func getChannel(c *gin.Context, info *relaycommon.RelayInfo, retryParam *service return channel, nil } +func applyIntelligentRouteNode(c *gin.Context, info *relaycommon.RelayInfo, channel *model.Channel, node hosttypes.IntelligentRouteNode) *types.NewAPIError { + if setupErr := middleware.SetupContextForSelectedChannel(c, channel, node.Model); setupErr != nil { + return setupErr + } + info.InitChannelMeta(c) + info.SetExecutionModelName(node.Model) + return nil +} + func shouldRetry(c *gin.Context, openaiErr *types.NewAPIError, retryTimes int) bool { if openaiErr == nil { return false diff --git a/docs/intelligent-routing-shadow-rollout.md b/docs/intelligent-routing-shadow-rollout.md new file mode 100644 index 000000000000..27fd006a957a --- /dev/null +++ b/docs/intelligent-routing-shadow-rollout.md @@ -0,0 +1,123 @@ +# Intelligent Routing Rollout and Operations + +## Runtime contract + +Intelligent routing is disabled by default and runs only for OpenAI-compatible Chat Completions and Responses text endpoints. Image, audio, embedding, rerank, realtime, Claude-native, Gemini-native, and task endpoints keep their existing routing and billing behavior. + +The client-facing model remains the model selected in the request. Live routing rewrites only the upstream execution request, normalizes the returned `model` field to the requested identifier, and bills against the successful execution model. It never automatically falls back to the requested model unless that model is independently present in the eligible candidate plan. + +Routing details remain backend-only under `other.admin_info.intelligent_routing`; ordinary user log views remove `admin_info`. + +## Configuration + +Published administrator policies and scoped rollouts are managed through the root-only `/api/intelligent-routing` endpoints. Drafts are validated and checksummed before publication; published versions are immutable, and rollback creates a new version. A rollout selects a published version, `shadow` or `live` mode, user/token group allowlists, and a deterministic traffic percentage guarded by a revision number. + +Instances refresh the durable rollout snapshot at startup and periodically. The stable bucket uses the policy version, account, and token so the same caller remains in the same cohort. If no durable rollout exists, the legacy global configuration below remains active for backward compatibility. + +Configure the registered `intelligent_routing_setting` object through the system options API or administrator settings storage: + +- `enabled`: enables planning. +- `shadow_only`: when `true`, records plans without changing execution; when `false`, executes the plan. +- `policy_version`: positive version stored with every decision. +- `max_attempts`: total upstream attempts, default `4`. +- `max_endpoints_per_model`: endpoint attempts per model, default `2`. +- `non_stream_budget`: Go duration in nanoseconds, default `30000000000`. +- `stream_first_byte_budget`: Go duration in nanoseconds, default `12000000000`. +- `max_cost_multiplier`: cumulative expected-cost ceiling relative to the first node, default `2.5`. +- `quality_thresholds`: task-to-probability map. +- `models`: candidate model policies. + +Example: + +```json +{ + "enabled": true, + "shadow_only": true, + "policy_version": 1, + "max_attempts": 4, + "max_endpoints_per_model": 2, + "non_stream_budget": 30000000000, + "stream_first_byte_budget": 12000000000, + "max_cost_multiplier": 2.5, + "quality_thresholds": { + "translation": 0.88, + "summary": 0.88, + "general": 0.90, + "code": 0.93, + "extraction": 0.94, + "reasoning": 0.95, + "json_schema": 0.97, + "tool": 0.98 + }, + "models": [ + { + "model": "deepseek/deepseek-chat", + "tier": 1, + "input_price": 0.28, + "output_price": 0.42, + "context_limit": 65536, + "capabilities": ["tools", "json_schema"] + } + ] +} +``` + +Routing prices use one consistent per-million-token unit and control candidate ordering. Configure the same execution models in the existing model-price/model-ratio billing settings; settlement uses those billing settings for the actual successful execution model. + +## Selection and fallback + +The router performs deterministic task classification, capability and 70%-context filtering, task-specific quality filtering, and expected-cost ordering. Cold-start quality priors are `0.88`, `0.92`, `0.96`, and `0.99` for tiers L0 through L3. After 30 model/task observations, beta-smoothed observed quality replaces the prior. + +Endpoint health uses a rolling 60-second window: + +- fewer than 20 observations: `PROBATION`; +- at least 99% success: `HEALTHY`; +- 95% through below 99% success: `DEGRADED`; +- below 95% success: `OPEN` and excluded until the rolling window expires. + +The execution sequence permits no more than four attempts, two endpoints per model, the configured elapsed-time budget, and 2.5 times the first candidate's expected cost. Reaching the time or cost limit skips directly to at most one remaining highest-success final candidate. + +Non-streaming responses are validated before client commitment for non-empty output, truncation, JSON output, declared tool names, and JSON tool arguments. A validation failure moves to the next route node. Streaming requests retry only before response commitment. + +## Session stickiness + +Clients may send `X-Session-ID`; otherwise the backend derives an account-scoped fingerprint from the first user message. A successful route is preferred for 30 minutes when the task is unchanged, the endpoint is not degraded/open, and its expected cost is no more than 1.15 times the cheapest qualified route. Two consecutive response-validation failures invalidate the sticky route. + +The session identifier and fingerprint are not returned to clients. + +## Audit + +Administrator routing audit contains: + +- policy version, requested model, execution model, shadow/live state; +- ordered candidate models, channels, quality probabilities, costs, and reason codes; +- every attempted node, outcome, bounded failure reason, and latency; +- final attempt index and planning error when present. + +The ordinary consume log continues to hold actual prompt/completion tokens and charged quota. Quota saturation remains under the adjacent `admin_info.quota_saturation` marker. + +## Rollout + +1. Configure candidate and billing prices with `enabled=false`. +2. Enable `shadow_only=true` and inspect candidate eligibility, savings, and no-route errors. +3. Set `shadow_only=false` for a controlled group after shadow results pass. +4. Monitor first-route success, multiple attempts, final failures, latency, actual charged quota, and quality-validation failures. +5. Increase `policy_version` for every policy change. + +## Rollback + +Set `intelligent_routing_setting.enabled=false`. The existing channel selector resumes immediately without a restart or frontend change. Historical routing audit retains its policy version. + +## Verification + +```powershell +$env:GOCACHE="$PWD\.gocache" +go test -race ./service/intelligent_routing -count=1 +go test ./setting/intelligent_routing_setting ./service/intelligent_routing ./model ./controller ./service ./middleware ./relay/channel/openai -count=1 +go test ./... -count=1 +Push-Location relaykit +$env:GOWORK="off" +go build ./... +Pop-Location +git diff --check +``` diff --git a/docs/superpowers/plans/2026-08-17-intelligent-routing-shadow-core.md b/docs/superpowers/plans/2026-08-17-intelligent-routing-shadow-core.md new file mode 100644 index 000000000000..60d1487a8750 --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-intelligent-routing-shadow-core.md @@ -0,0 +1,466 @@ +# Intelligent Routing Shadow Core Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the first production-safe increment of cost-optimized intelligent routing: deterministic request features, capability filtering, expected-cost route planning, and shadow-mode audit output without changing live request execution. + +**Architecture:** A focused `service/intelligent_routing` package produces immutable route plans from normalized request features and an injected candidate catalog. `controller/relay.go` invokes it after request validation and token estimation, records the shadow plan on `RelayInfo`, and leaves the existing channel-selection and retry path unchanged. Later plans can consume the same route plan for live cross-model execution. + +**Tech Stack:** Go 1.22+, Gin, existing model/channel cache, existing ratio settings, testify `require`/`assert`. + +**Spec:** `docs/superpowers/specs/2026-08-17-cost-optimized-intelligent-routing-design.md` + +## Global Constraints + +- Keep `relaykit/` independently buildable and do not import root-module packages from it. +- Use `common.Marshal`, `common.Unmarshal`, `common.UnmarshalJsonStr`, or `common.DecodeJson` for JSON operations. +- Preserve SQLite, MySQL 5.7.8+, and PostgreSQL 9.6 compatibility. +- Use checked quota conversion helpers for every value that can become a charge; surface clamps through the existing audit path. +- Preserve the requested model as `RelayInfo.OriginModelName`; shadow routing must not alter live execution. +- New tests use `require` for setup/fatal assertions and `assert` for value assertions. +- Do not modify protected project identity, attribution, package paths, or branding. + +--- + +### Task 1: Versioned routing configuration + +**Files:** +- Create: `setting/intelligent_routing_setting/config.go` +- Create: `setting/intelligent_routing_setting/config_test.go` +- Modify: `setting/operation_setting.go` + +**Interfaces:** +- Produces: `intelligent_routing_setting.Config`, `Get() Config`, `Update(Config) error`, `Enabled() bool`. +- Consumes: existing option registration pattern in `setting/operation_setting.go`. + +- [ ] **Step 1: Write failing configuration tests** + +Cover exact normalization: disabled defaults, policy version `1`, four attempts, two endpoints per model, 30-second non-stream budget, 12-second stream-first-byte budget, 2.5 cost multiplier, and task thresholds from the spec. Assert rejection of negative budgets, thresholds outside `[0,1]`, duplicate model entries, and tiers outside `0..3`. + +```go +func TestNormalizeConfigAppliesSafeDefaults(t *testing.T) { + got, err := Normalize(Config{Enabled: true}) + require.NoError(t, err) + assert.Equal(t, 1, got.PolicyVersion) + assert.Equal(t, 4, got.MaxAttempts) + assert.Equal(t, 2, got.MaxEndpointsPerModel) + assert.Equal(t, 30*time.Second, got.NonStreamBudget) + assert.InDelta(t, 0.98, got.QualityThresholds[TaskTool], 0.0001) +} +``` + +- [ ] **Step 2: Run the focused test and verify failure** + +Run: `go test ./setting/intelligent_routing_setting -run TestNormalizeConfig -count=1` + +Expected: FAIL because the package and `Normalize` do not exist. + +- [ ] **Step 3: Implement immutable configuration snapshots** + +Define `TaskType`, `ModelPolicy`, and `Config`. Store the normalized configuration in `atomic.Pointer[Config]`; `Get` returns a value copy. `Update` normalizes a copy before publishing it. Use explicit code defaults rather than GORM boolean tags. + +```go +type ModelPolicy struct { + Model string + Tier int + InputPrice float64 + OutputPrice float64 + ContextLimit int + Capabilities []string +} + +type Config struct { + Enabled bool + ShadowOnly bool + PolicyVersion int + MaxAttempts int + MaxEndpointsPerModel int + NonStreamBudget time.Duration + StreamFirstByteBudget time.Duration + MaxCostMultiplier float64 + QualityThresholds map[TaskType]float64 + Models []ModelPolicy +} +``` + +- [ ] **Step 4: Register the setting and rerun tests** + +Register a single serialized option named `IntelligentRoutingConfig`; decode it through `common.UnmarshalJsonStr`, call `Update`, and serialize snapshots with `common.Marshal`. + +Run: `go test ./setting/intelligent_routing_setting ./setting -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```powershell +git add setting/intelligent_routing_setting setting/operation_setting.go +git commit -m "feat: add intelligent routing configuration" +``` + +### Task 2: Deterministic request feature extraction + +**Files:** +- Create: `service/intelligent_routing/features.go` +- Create: `service/intelligent_routing/features_test.go` + +**Interfaces:** +- Consumes: `dto.Request`, `types.RelayFormat`, estimated prompt tokens, request path, stream flag. +- Produces: `ExtractFeatures(Input) Features`, `DeriveRequirements(Features) Requirements`. + +- [ ] **Step 1: Write table tests for observable request categories** + +Use explicit OpenAI requests for short translation, summary, tool call, strict response format, long context, and code generation. Assert task type, required capabilities, token band, and minimum tier. Do not assert private keyword lists. + +```go +func TestExtractFeaturesRequiresTools(t *testing.T) { + req := &dto.GeneralOpenAIRequest{ + Model: "client-model", + Messages: []dto.Message{{Role: "user", Content: "check the weather"}}, + Tools: []dto.Tool{{Type: "function"}}, + } + got := ExtractFeatures(Input{Request: req, PromptTokens: 24}) + assert.Equal(t, TaskTool, got.Task) + assert.True(t, got.HasTools) + assert.GreaterOrEqual(t, got.MinimumTier, 2) +} +``` + +- [ ] **Step 2: Verify the tests fail** + +Run: `go test ./service/intelligent_routing -run 'TestExtractFeatures|TestDeriveRequirements' -count=1` + +Expected: FAIL because feature extraction is undefined. + +- [ ] **Step 3: Implement direct type-switch extraction** + +Implement one top-level type switch over supported request DTOs. Classify hard protocol features first; use normalized text hints only for translation, summary, extraction, code, math, and general tasks. Return general task for ambiguous inputs. Keep helpers only for stable concepts such as combined request text and capability derivation. + +```go +type Features struct { + Task TaskType + PromptTokens int + MaxOutputTokens int + ContextUtilization float64 + HasTools bool + RequiresJSONSchema bool + HasImage bool + IsStream bool + MinimumTier int +} + +type Requirements struct { + Capabilities map[Capability]bool + MinimumTier int + ContextNeeded int +} +``` + +- [ ] **Step 4: Run focused and package tests** + +Run: `go test ./service/intelligent_routing -count=1` + +Expected: PASS with deterministic results on every table row. + +- [ ] **Step 5: Commit** + +```powershell +git add service/intelligent_routing/features.go service/intelligent_routing/features_test.go +git commit -m "feat: extract intelligent routing features" +``` + +### Task 3: Candidate catalog adapter + +**Files:** +- Create: `service/intelligent_routing/catalog.go` +- Create: `service/intelligent_routing/catalog_test.go` +- Modify: `model/channel_cache.go` + +**Interfaces:** +- Consumes: normalized routing config and a read-only snapshot returned by `model.ListEnabledChannelsForRouting(group, requestPath string) []*model.Channel`. +- Produces: `Catalog.Build(group, requestPath string) []Candidate`. + +- [ ] **Step 1: Write failing catalog tests** + +Build channel fixtures that cover enabled/disabled status, group membership, model mapping, Advanced Custom path support, configured model tier, and missing price. Assert disabled, incompatible, and unpriced nodes are absent. + +```go +func TestCatalogExcludesUnpricedAndDisabledCandidates(t *testing.T) { + catalog := NewCatalog(config, fakeChannels) + got := catalog.Build("default", "/v1/chat/completions") + require.Len(t, got, 1) + assert.Equal(t, "cheap-model", got[0].Model) + assert.Equal(t, 7, got[0].ChannelID) +} +``` + +- [ ] **Step 2: Verify failure** + +Run: `go test ./service/intelligent_routing -run TestCatalog -count=1` + +Expected: FAIL because the catalog and channel snapshot API do not exist. + +- [ ] **Step 3: Add a read-only channel-cache snapshot** + +Under the existing channel cache read lock, return cloned enabled channels that match the group and request path. Do not expose internal cache maps and do not change existing random selection. + +```go +func ListEnabledChannelsForRouting(group, requestPath string) []*Channel +``` + +- [ ] **Step 4: Implement catalog normalization** + +Expand configured model policies into candidate nodes only where a channel serves the model directly or through existing normalized-name matching. Copy values into the candidate so a later cache refresh cannot mutate an active plan. + +```go +type Candidate struct { + Model string + ChannelID int + Tier int + InputPrice float64 + OutputPrice float64 + ContextLimit int + Capabilities map[Capability]bool + ResponseTimeMS int +} +``` + +- [ ] **Step 5: Run model and routing tests** + +Run: `go test ./model ./service/intelligent_routing -count=1` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```powershell +git add model/channel_cache.go service/intelligent_routing/catalog.go service/intelligent_routing/catalog_test.go +git commit -m "feat: build intelligent routing candidate catalog" +``` + +### Task 4: Constrained expected-cost planner + +**Files:** +- Create: `service/intelligent_routing/planner.go` +- Create: `service/intelligent_routing/planner_test.go` + +**Interfaces:** +- Consumes: `PlanInput{RequestedModel, Features, Requirements, Candidates, PolicyVersion}`. +- Produces: `Plan(PlanInput) (RoutePlan, error)` with ordered immutable `RouteNode` values. + +- [ ] **Step 1: Write exact planner contract tests** + +Cover capability rejection, context rejection at 70% utilization, task quality thresholds, cheapest-qualified selection, no-qualified fallback to highest predicted success, same-model endpoint grouping, maximum two endpoints per model, four total nodes, and reserved highest-success final node. + +```go +func TestPlanChoosesCheapestCandidateMeetingQualityThreshold(t *testing.T) { + got, err := Plan(PlanInput{ + RequestedModel: "client-model", + Features: Features{Task: TaskGeneral, PromptTokens: 100, MaxOutputTokens: 50}, + Candidates: []Candidate{ + {Model: "cheap", ChannelID: 1, InputPrice: 1, OutputPrice: 2, PredictedSuccess: .92}, + {Model: "premium", ChannelID: 2, InputPrice: 8, OutputPrice: 16, PredictedSuccess: .99}, + }, + QualityThreshold: .90, + }) + require.NoError(t, err) + assert.Equal(t, "cheap", got.Nodes[0].Model) + assert.Equal(t, "premium", got.Nodes[len(got.Nodes)-1].Model) +} +``` + +- [ ] **Step 2: Verify planner tests fail** + +Run: `go test ./service/intelligent_routing -run TestPlan -count=1` + +Expected: FAIL because `Plan` is undefined. + +- [ ] **Step 3: Implement threshold filtering and cost calculation** + +Use decimal arithmetic for price products. The planner's estimates remain decimals until rendered for audit; no estimated quota is cast to `int`. Calculate input, predicted output, cache, and retry-risk components independently. + +```go +type RouteNode struct { + Model string + ChannelID int + Tier int + PredictedSuccess float64 + ExpectedCost decimal.Decimal + ReasonCodes []string +} + +type RoutePlan struct { + RequestedModel string + PolicyVersion int + Nodes []RouteNode + MaxAttempts int + MaxCostMultiplier float64 +} +``` + +- [ ] **Step 4: Implement stable lexicographic ordering** + +Sort by health tier, expected total cost, response time, failure rate, and channel ID. Group at most two endpoints for the selected model before moving to the next qualified model. Deduplicate `(model, channel)` and preserve the highest-success candidate for the final slot. + +- [ ] **Step 5: Run routing tests with the race detector** + +Run: `go test -race ./service/intelligent_routing -count=1` + +Expected: PASS and no race report. + +- [ ] **Step 6: Commit** + +```powershell +git add service/intelligent_routing/planner.go service/intelligent_routing/planner_test.go +git commit -m "feat: plan constrained low cost routes" +``` + +### Task 5: Shadow-plan integration in text relay + +**Files:** +- Modify: `relay/common/relay_info.go` +- Modify: `controller/relay.go` +- Create: `controller/intelligent_routing_shadow_test.go` + +**Interfaces:** +- Consumes: validated `dto.Request`, prompt-token estimate, current group, request path, and routing config. +- Produces: `RelayInfo.IntelligentRoutePlan *intelligent_routing.RoutePlan`; live selected channel and `OriginModelName` remain unchanged. + +- [ ] **Step 1: Write failing controller tests** + +Build a Gin context with a reusable request body and a configured candidate catalog. Assert shadow mode produces a plan after validation, leaves `OriginModelName` unchanged, leaves current channel context unchanged, and silently records a typed planning error when no candidate is eligible. + +```go +func TestBuildShadowRoutePlanDoesNotChangeLiveModel(t *testing.T) { + info := &relaycommon.RelayInfo{OriginModelName: "client-model", Request: request} + err := buildShadowRoutePlan(ctx, info, 120) + require.NoError(t, err) + require.NotNil(t, info.IntelligentRoutePlan) + assert.Equal(t, "client-model", info.OriginModelName) + assert.Equal(t, originalChannelID, common.GetContextKeyInt(ctx, constant.ContextKeyChannelId)) +} +``` + +- [ ] **Step 2: Verify failure** + +Run: `go test ./controller -run TestBuildShadowRoutePlan -count=1` + +Expected: FAIL because the integration function and RelayInfo field do not exist. + +- [ ] **Step 3: Add the shadow plan to RelayInfo** + +Add the pointer field and a string error field used only for administrative diagnostics. Do not place routing types in `relaykit`. + +- [ ] **Step 4: Build the plan after token estimation** + +Invoke `buildShadowRoutePlan` after `SetEstimatePromptTokens` and before pre-consume. Guard it with `Enabled && ShadowOnly`. Planning errors log a request-correlated warning and do not affect live relay behavior. + +- [ ] **Step 5: Prove live relay behavior is unchanged** + +Run: `go test ./controller ./middleware ./relay/... -count=1` + +Expected: PASS, including existing channel retry and response model tests. + +- [ ] **Step 6: Commit** + +```powershell +git add relay/common/relay_info.go controller/relay.go controller/intelligent_routing_shadow_test.go +git commit -m "feat: add shadow intelligent route planning" +``` + +### Task 6: Administrative audit payload and metrics + +**Files:** +- Modify: `service/log_info_generate.go` +- Create: `service/intelligent_routing_audit_test.go` +- Create: `service/intelligent_routing/metrics.go` +- Create: `service/intelligent_routing/metrics_test.go` + +**Interfaces:** +- Consumes: `RelayInfo.IntelligentRoutePlan` and the actual successful channel/model metadata. +- Produces: `other.admin_info.intelligent_routing` and aggregate `Metrics.Observe(Observation)`. + +- [ ] **Step 1: Write failing admin-only audit tests** + +Assert exact keys: `policy_version`, `requested_model`, `shadow`, `candidates`, `predicted_success`, `expected_cost`, and `reason_codes`. Assert the payload is nested under `admin_info`, alongside rather than replacing `quota_saturation`. + +- [ ] **Step 2: Verify audit tests fail** + +Run: `go test ./service -run TestIntelligentRoutingAudit -count=1` + +Expected: FAIL because no routing audit is attached. + +- [ ] **Step 3: Attach a bounded audit payload** + +Serialize at most four nodes and at most eight reason codes per node. Reuse the existing `admin_info` map and never expose candidate details through non-admin log views. + +- [ ] **Step 4: Add deterministic metric aggregation** + +Implement counters for planned requests, no-route requests, candidate tier distribution, expected saving, and planning latency. Do not use sleeps or timing assertions; inject elapsed duration into `Observe` tests. + +- [ ] **Step 5: Run service and routing tests** + +Run: `go test ./service ./service/intelligent_routing -count=1` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```powershell +git add service/log_info_generate.go service/intelligent_routing_audit_test.go service/intelligent_routing/metrics.go service/intelligent_routing/metrics_test.go +git commit -m "feat: audit shadow routing decisions" +``` + +### Task 7: Full verification and rollout documentation + +**Files:** +- Create: `docs/intelligent-routing-shadow-rollout.md` +- Modify: `.env.example` + +**Interfaces:** +- Consumes: completed shadow planner and configuration. +- Produces: operator instructions for disabled, shadow, and rollback states. + +- [ ] **Step 1: Document exact rollout controls** + +Document the serialized setting, default-disabled behavior, shadow metrics, candidate configuration example, log query fields, and rollback action (`Enabled=false`). State that this phase never changes the live execution model. + +- [ ] **Step 2: Run formatting and focused tests** + +Run: `gofmt -w setting/intelligent_routing_setting service/intelligent_routing controller/intelligent_routing_shadow_test.go service/intelligent_routing_audit_test.go` + +Run: `go test ./setting/intelligent_routing_setting ./service/intelligent_routing ./model ./controller ./service ./middleware -count=1` + +Expected: PASS. + +- [ ] **Step 3: Run repository-wide verification** + +Run: `go test ./... -count=1` + +Expected: PASS. + +- [ ] **Step 4: Verify relaykit independence** + +Run from `relaykit/`: `$env:GOWORK='off'; go build ./...` + +Expected: exit status 0. + +- [ ] **Step 5: Inspect the final diff and commit** + +Run: `git diff --check` + +Expected: no output, exit status 0. + +```powershell +git add .env.example docs/intelligent-routing-shadow-rollout.md +git commit -m "docs: add intelligent routing shadow rollout" +``` + +## Follow-up implementation plans + +The shadow core is independently deployable and measurable. After its observation data is validated, create separate plans for: + +1. Live same-model price-aware endpoint selection and circuit breaking. +2. Live cross-model execution with body rewriting, actual-model billing, response normalization, and bounded fallback. +3. Learned quality prediction, calibration, session stickiness, semantic judging, and progressive model onboarding. + diff --git a/docs/superpowers/plans/2026-08-17-nailong-cost-routing-core.md b/docs/superpowers/plans/2026-08-17-nailong-cost-routing-core.md new file mode 100644 index 000000000000..3905e2e1a962 --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-nailong-cost-routing-core.md @@ -0,0 +1,742 @@ +# Nailong Cost Routing Core Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add an opt-in, deterministic cross-model cost router to New API that preserves requested, actual, and upstream model identities through authorization, channel selection, billing, retries, responses, and audit logs. + +**Architecture:** A focused `service/cost_router` package evaluates enabled database-backed rules before `middleware.Distribute` selects a channel. The selected actual model flows through the existing New API relay and billing pipeline, while explicit context and `RelayInfo` fields retain the requested model. API-key defaults and per-request overrides determine whether routing is strict or cost optimized. + +**Tech Stack:** Go 1.22+, Gin, GORM v2, testify, React 19, TypeScript, Bun, SQLite/MySQL/PostgreSQL-compatible migrations + +**Spec:** `docs/superpowers/specs/2026-08-17-nailong-cost-routing-design.md` + +## Global Constraints + +- Preserve support for SQLite, MySQL 5.7.8+, and PostgreSQL 9.6+. +- Use `common.Marshal`, `common.Unmarshal`, and `common.UnmarshalJsonStr` for JSON operations. +- Preserve New API and QuantumNous attribution and licensing information. +- Cross-model routing is opt-in; the default mode is exactly `strict`. +- Authorize both the requested and actual models before contacting an upstream provider. +- Bill from the actual model's immutable request-time price snapshot. +- Never store full prompts, authorization headers, or upstream secrets in routing records. +- Never change models after the first response byte has been emitted. +- Keep `relaykit/` independently buildable and do not add root-module dependencies to it. +- New backend tests use `testify/require` and `testify/assert`. + +## Scope Boundary + +This plan delivers the routing-core vertical slice: rule persistence, deterministic selection, API-key preference, middleware integration, model identity propagation, billing correctness, fallback, response disclosure, backend management APIs, and focused user controls. Invitation registration, complete brand redesign, dynamic channel health scoring, dashboards, alerting, and production deployment automation remain separate implementation plans because each can be reviewed and released independently. + +## File Structure + +- `model/routing_rule.go`: persistent rule and routing-decision records plus validated JSON accessors. +- `model/token.go`: API-key routing defaults. +- `model/main.go`: cross-database migration registration. +- `service/cost_router/types.go`: public routing request, result, mode, rule, and capability types. +- `service/cost_router/router.go`: deterministic rule matching and cheapest-capable-model selection. +- `service/cost_router/store.go`: database rule loader behind an interface suitable for tests and future caching. +- `middleware/cost_router.go`: request preference parsing, authorization, model rewrite, and routing context setup. +- `constant/context_key.go`: strongly named routing context keys. +- `relay/common/relay_info.go`: requested/actual/routing identity snapshot used by billing, retry, and logging. +- `service/log_info_generate.go`: routing audit fields in consume logs. +- `controller/relay.go`: two-stage actual-model then requested-model fallback. +- `controller/routing_rule.go`: administrator rule CRUD and dry-run endpoint. +- `router/api-router.go`: management API registration. +- `controller/token.go`: read and write API-key routing defaults. +- `web/src/pages/Token/index.jsx` and token form components discovered in Task 9: opt-in controls and disclosure. + +--- + +### Task 1: Persist Validated Routing Rules and Decisions + +**Files:** +- Create: `model/routing_rule.go` +- Create: `model/routing_rule_test.go` +- Modify: `model/main.go` + +**Interfaces:** +- Produces: `model.RoutingRule`, `model.RoutingDecision`, `model.ListEnabledRoutingRules() ([]RoutingRule, error)`, `(*RoutingRule).ReplacementModels() ([]RoutingReplacement, error)`. +- Consumes: `common.Marshal`, `common.UnmarshalJsonStr`, `model.DB`. + +- [ ] **Step 1: Write failing model and migration tests** + +```go +func TestRoutingRuleReplacementModelsRejectsInvalidCost(t *testing.T) { + raw := `[{"model":"deepseek-v3","priority":100,"input_cost":-1,"output_cost":2}]` + rule := RoutingRule{ReplacementModelsJSON: raw} + + _, err := rule.ReplacementModels() + + require.ErrorContains(t, err, "input_cost must be non-negative") +} + +func TestRoutingDecisionAutoMigrate(t *testing.T) { + db := newTestDB(t) + require.NoError(t, db.AutoMigrate(&RoutingRule{}, &RoutingDecision{})) + require.True(t, db.Migrator().HasTable(&RoutingRule{})) + require.True(t, db.Migrator().HasTable(&RoutingDecision{})) +} +``` + +- [ ] **Step 2: Run the tests and verify the missing types fail compilation** + +Run: `go test ./model -run 'TestRouting(RuleReplacementModelsRejectsInvalidCost|DecisionAutoMigrate)' -count=1` + +Expected: FAIL because `RoutingRule` and `RoutingDecision` are undefined. + +- [ ] **Step 3: Implement the models and strict JSON validation** + +```go +type RoutingReplacement struct { + Model string `json:"model"` + Priority int `json:"priority"` + InputCost float64 `json:"input_cost"` + OutputCost float64 `json:"output_cost"` +} + +type RoutingRule struct { + Id int `json:"id"` + Name string `json:"name" gorm:"size:128;not null"` + Status int `json:"status" gorm:"index"` + Priority int `json:"priority" gorm:"index"` + RequestedModelPattern string `json:"requested_model_pattern" gorm:"size:191;not null;index"` + ReplacementModelsJSON string `json:"replacement_models_json" gorm:"type:text;not null"` + EndpointsJSON string `json:"endpoints_json" gorm:"type:text"` + CapabilitiesJSON string `json:"capabilities_json" gorm:"type:text"` + MaxContextTokens int `json:"max_context_tokens"` + MaxEstimatedCost float64 `json:"max_estimated_cost"` + FallbackToRequested bool `json:"fallback_to_requested"` + UserGroupsJSON string `json:"user_groups_json" gorm:"type:text"` + EffectiveFrom int64 `json:"effective_from" gorm:"bigint"` + EffectiveUntil int64 `json:"effective_until" gorm:"bigint"` + CreatedAt int64 `json:"created_at" gorm:"bigint"` + UpdatedAt int64 `json:"updated_at" gorm:"bigint"` +} + +type RoutingDecision struct { + Id int `json:"id"` + RequestId string `json:"request_id" gorm:"size:64;index"` + UserId int `json:"user_id" gorm:"index"` + TokenId int `json:"token_id" gorm:"index"` + RequestedModel string `json:"requested_model" gorm:"size:191;index"` + ActualModel string `json:"actual_model" gorm:"size:191;index"` + RoutingMode string `json:"routing_mode" gorm:"size:32"` + RuleId int `json:"rule_id" gorm:"index"` + DecisionReason string `json:"decision_reason" gorm:"size:255"` + FallbackModel string `json:"fallback_model" gorm:"size:191"` + FallbackTriggered bool `json:"fallback_triggered"` + EstimatedOriginalCost float64 `json:"estimated_original_cost"` + EstimatedActualCost float64 `json:"estimated_actual_cost"` + ActualCost float64 `json:"actual_cost"` + ChannelId int `json:"channel_id" gorm:"index"` + RequestFeaturesJSON string `json:"request_features_json" gorm:"type:text"` + CreatedAt int64 `json:"created_at" gorm:"bigint;index"` +} +``` + +Validate empty model names, duplicate candidates, NaN/Inf values, negative prices, invalid JSON, invalid effective windows, and replacement lists larger than 32. Add both models to normal and fast migration lists in `model/main.go`. + +- [ ] **Step 4: Run model tests and migration smoke tests** + +Run: `go test ./model -run 'TestRouting' -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit the persistence boundary** + +```bash +git add model/routing_rule.go model/routing_rule_test.go model/main.go +git commit -m "feat: persist cost routing rules" +``` + +### Task 2: Build the Deterministic Cost Router + +**Files:** +- Create: `service/cost_router/types.go` +- Create: `service/cost_router/store.go` +- Create: `service/cost_router/router.go` +- Create: `service/cost_router/router_test.go` + +**Interfaces:** +- Produces: `costrouter.Mode`, `costrouter.Request`, `costrouter.Result`, `costrouter.Router`, `costrouter.New(store RuleStore) *Router`, `(*Router).Route(context.Context, Request) (Result, error)`. +- Consumes: `model.RoutingRule`, `model.ListEnabledRoutingRules`. + +- [ ] **Step 1: Write failing table-driven routing tests** + +```go +func TestRouterSelectsCheapestCapableReplacement(t *testing.T) { + router := New(staticStore{rules: []Rule{{ + ID: 7, RequestedModelPattern: "gpt-*", Endpoints: []string{"chat.completions"}, + Replacements: []Replacement{ + {Model: "cheap-text", InputCost: 0.1, OutputCost: 0.2, Capabilities: CapabilityText}, + {Model: "vision-model", InputCost: 0.2, OutputCost: 0.4, Capabilities: CapabilityText | CapabilityVision}, + }, + }}}) + + got, err := router.Route(context.Background(), Request{ + Mode: ModeOptimizeCost, RequestedModel: "gpt-5-mini", Endpoint: "chat.completions", + RequiredCapabilities: CapabilityText, PromptTokens: 1000, EstimatedOutputTokens: 500, + AuthorizedModels: map[string]bool{"gpt-5-mini": true, "cheap-text": true, "vision-model": true}, + AvailableModels: map[string]bool{"cheap-text": true, "vision-model": true}, + }) + + require.NoError(t, err) + assert.Equal(t, "cheap-text", got.ActualModel) + assert.Equal(t, 7, got.RuleID) + assert.True(t, got.Substituted) +} +``` + +Add cases for strict mode, unmatched rules, unauthorized actual models, missing capabilities, insufficient context, unavailable models, cost ceiling, time windows, user groups, stable priority ordering, and fallback to the requested model when no replacement qualifies. + +- [ ] **Step 2: Run the router test and verify it fails** + +Run: `go test ./service/cost_router -run TestRouter -count=1` + +Expected: FAIL because the package does not exist. + +- [ ] **Step 3: Implement focused public types and selection logic** + +```go +type Mode string + +const ( + ModeStrict Mode = "strict" + ModeOptimizeCost Mode = "optimize_cost" +) + +type Request struct { + Mode Mode + RequestedModel string + Endpoint string + UserGroup string + RequiredCapabilities Capability + ContextTokens int + PromptTokens int + EstimatedOutputTokens int + MaxCost float64 + AuthorizedModels map[string]bool + AvailableModels map[string]bool + Now time.Time +} + +type Result struct { + RequestedModel string + ActualModel string + Mode Mode + RuleID int + Reason string + FallbackModel string + Substituted bool + EstimatedOriginalCost float64 + EstimatedActualCost float64 +} +``` + +Use `path.Match` only after rejecting malformed patterns and patterns containing path separators. Sort candidates by estimated cost ascending, replacement priority descending, and model name ascending so decisions remain deterministic. + +- [ ] **Step 4: Run router tests and package tests** + +Run: `go test ./service/cost_router -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit the routing engine** + +```bash +git add service/cost_router +git commit -m "feat: add deterministic cost router" +``` + +### Task 3: Add API-Key Routing Defaults to Authentication Context + +**Files:** +- Modify: `model/token.go` +- Modify: `constant/context_key.go` +- Modify: `middleware/auth.go` +- Modify: `controller/token.go` +- Test: `controller/token_test.go` +- Test: `middleware/auth_test.go` + +**Interfaces:** +- Produces context keys `ContextKeyTokenRoutingMode`, `ContextKeyTokenRoutingMaxCost`, `ContextKeyTokenRoutingFallback`. +- Consumes `costrouter.ModeStrict` and `costrouter.ModeOptimizeCost` only in controller validation; the model stores strings to avoid package cycles. + +- [ ] **Step 1: Write failing token validation and auth-context tests** + +```go +func TestUpdateTokenRejectsUnknownRoutingMode(t *testing.T) { + token := model.Token{RoutingMode: "hidden_swap"} + err := validateTokenRouting(&token) + require.ErrorContains(t, err, "routing_mode must be strict or optimize_cost") +} + +func TestTokenAuthPublishesRoutingDefaults(t *testing.T) { + c := authenticatedTokenContext(t, model.Token{ + RoutingMode: "optimize_cost", RoutingMaxCost: 0.02, RoutingFallback: true, + }) + assert.Equal(t, "optimize_cost", common.GetContextKeyString(c, constant.ContextKeyTokenRoutingMode)) + assert.InDelta(t, 0.02, common.GetContextKeyFloat64(c, constant.ContextKeyTokenRoutingMaxCost), 0.000001) + assert.True(t, common.GetContextKeyBool(c, constant.ContextKeyTokenRoutingFallback)) +} +``` + +- [ ] **Step 2: Run focused tests and verify failure** + +Run: `go test ./controller ./middleware -run 'Test(UpdateTokenRejectsUnknownRoutingMode|TokenAuthPublishesRoutingDefaults)' -count=1` + +Expected: FAIL because routing fields and context keys are absent. + +- [ ] **Step 3: Add fields, validation, serialization, and authentication context** + +```go +RoutingMode string `json:"routing_mode" gorm:"size:32"` +RoutingMaxCost float64 `json:"routing_max_cost"` +RoutingFallback bool `json:"routing_fallback"` +``` + +Normalize an empty mode to `strict`; reject non-finite or negative maximum cost; require explicit disclosure acceptance in the existing token update request before accepting `optimize_cost`. Ensure token list and update responses expose these non-secret settings. + +- [ ] **Step 4: Run token and middleware test suites** + +Run: `go test ./controller ./middleware -run 'Token|Routing' -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit API-key defaults** + +```bash +git add model/token.go constant/context_key.go middleware/auth.go controller/token.go controller/token_test.go middleware/auth_test.go +git commit -m "feat: add token cost routing preferences" +``` + +### Task 4: Route Before Channel Distribution Without Losing the Requested Model + +**Files:** +- Create: `middleware/cost_router.go` +- Create: `middleware/cost_router_test.go` +- Modify: `middleware/distributor.go` +- Modify: `router/relay-router.go` +- Modify: `constant/context_key.go` + +**Interfaces:** +- Produces: `middleware.CostRoute() gin.HandlerFunc` and routing context keys for requested model, actual model, mode, rule ID, reason, fallback model, and estimated saving. +- Consumes: `costrouter.Router.Route`, authenticated token routing defaults, reusable request body storage. + +- [ ] **Step 1: Write middleware tests for strict and optimized requests** + +```go +func TestCostRouteRewritesModelAndPreservesRequestedModel(t *testing.T) { + c, recorder := routingContext(t, `{"model":"gpt-5-mini","messages":[{"role":"user","content":"hi"}]}`) + setTokenRouting(c, "optimize_cost", 0, true) + handler := CostRouteWithRouter(fakeRouter{result: costrouter.Result{ + RequestedModel: "gpt-5-mini", ActualModel: "deepseek-v3", Mode: costrouter.ModeOptimizeCost, + RuleID: 9, Reason: "lowest_estimated_cost", FallbackModel: "gpt-5-mini", Substituted: true, + }}) + + handler(c) + + require.False(t, c.IsAborted()) + assert.Equal(t, "gpt-5-mini", common.GetContextKeyString(c, constant.ContextKeyRequestedModel)) + assert.Equal(t, "deepseek-v3", common.GetContextKeyString(c, constant.ContextKeyActualModel)) + assert.Contains(t, reusableBodyString(t, c), `"model":"deepseek-v3"`) + assert.Equal(t, http.StatusOK, recorder.Code) +} +``` + +Add tests for default strict mode, valid request-header override, invalid mode, invalid maximum cost, token restrictions, missing model, multipart requests that cannot be safely rewritten, and preservation of explicit request fields. + +- [ ] **Step 2: Run the middleware tests and verify failure** + +Run: `go test ./middleware -run TestCostRoute -count=1` + +Expected: FAIL because `CostRoute` is undefined. + +- [ ] **Step 3: Implement request parsing and safe rewrite** + +Register middleware in this order for supported JSON relay routes: + +```go +relayV1Router.Use(middleware.TokenAuth()) +relayV1Router.Use(middleware.ModelRequestRateLimit()) +relayV1Router.Use(middleware.CostRoute()) +httpRouter.Use(middleware.Distribute()) +``` + +Do not run cross-model replacement for realtime WebSocket, multipart image/audio/video, task-fetch, or endpoints without a safely parsed model in this first slice. Those paths remain strict and gain coverage in later compatibility work. Refactor `Distribute` so token model-limit validation is callable for both requested and actual model names. + +- [ ] **Step 4: Run middleware and relay-router tests** + +Run: `go test ./middleware ./router -run 'CostRoute|Distribute' -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit middleware integration** + +```bash +git add middleware/cost_router.go middleware/cost_router_test.go middleware/distributor.go router/relay-router.go constant/context_key.go +git commit -m "feat: route optimized models before distribution" +``` + +### Task 5: Preserve Three Model Identities Through Relay, Pricing, and Logs + +**Files:** +- Modify: `relay/common/relay_info.go` +- Modify: `relay/common/relay_info_test.go` +- Modify: `relay/helper/model_mapped.go` +- Modify: `service/log_info_generate.go` +- Modify: `service/log_info_generate_test.go` +- Modify: `service/token_counter.go` + +**Interfaces:** +- Produces `RelayInfo.RequestedModelName`, `RelayInfo.ActualModelName`, `RelayInfo.RoutingMode`, `RelayInfo.RoutingRuleID`, `RelayInfo.RoutingReason`, and `RelayInfo.RoutingFallbackModel`. +- Consumes routing context from Task 4. + +- [ ] **Step 1: Write failing relay identity and audit-log tests** + +```go +func TestGenRelayInfoSeparatesRequestedActualAndUpstreamModels(t *testing.T) { + c := relayContext(t) + common.SetContextKey(c, constant.ContextKeyRequestedModel, "gpt-5-mini") + common.SetContextKey(c, constant.ContextKeyActualModel, "deepseek-v3") + common.SetContextKey(c, constant.ContextKeyOriginalModel, "deepseek-v3") + + info, err := GenRelayInfo(c, types.RelayFormatOpenAI, requestFor("deepseek-v3"), nil) + + require.NoError(t, err) + assert.Equal(t, "gpt-5-mini", info.RequestedModelName) + assert.Equal(t, "deepseek-v3", info.ActualModelName) + assert.Equal(t, "deepseek-v3", info.OriginModelName) +} +``` + +Verify `GenerateTextOtherInfo` emits user-visible `requested_model`, `actual_model`, `routing_mode`, `routing_rule_id`, and `estimated_saving`, while upstream attempt details remain under `admin_info` where appropriate. + +- [ ] **Step 2: Run focused tests and verify failure** + +Run: `go test ./relay/common ./service -run 'ModelIdentit|RoutingAudit' -count=1` + +Expected: FAIL because new `RelayInfo` fields are missing. + +- [ ] **Step 3: Implement identity propagation and pricing invariants** + +Initialize routing fields in `GenRelayInfo`. Keep `OriginModelName` equal to `ActualModelName` so existing `ModelPriceHelper`, token estimation, channel retry, and settlement naturally use the actual model. `InitChannelMeta` begins with the actual model and existing channel mapping remains solely responsible for `UpstreamModelName`. + +Add this log shape: + +```go +if relayInfo.RoutingMode == "optimize_cost" { + other["requested_model"] = relayInfo.RequestedModelName + other["actual_model"] = relayInfo.ActualModelName + other["routing_mode"] = relayInfo.RoutingMode + other["routing_rule_id"] = relayInfo.RoutingRuleID + other["estimated_saving"] = relayInfo.EstimatedSaving +} +``` + +- [ ] **Step 4: Run relay, price, billing, and log tests** + +Run: `go test ./relay/common ./relay/helper ./service -run 'RelayInfo|Price|Billing|Log|Routing' -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit model identity propagation** + +```bash +git add relay/common/relay_info.go relay/common/relay_info_test.go relay/helper/model_mapped.go service/log_info_generate.go service/log_info_generate_test.go service/token_counter.go +git commit -m "feat: preserve routed model identities" +``` + +### Task 6: Add Bounded Original-Model Fallback With Correct Billing Lifecycle + +**Files:** +- Modify: `controller/relay.go` +- Create: `controller/relay_cost_routing_test.go` +- Modify: `service/billing_session.go` +- Modify: `service/billing_session_test.go` +- Modify: `model/routing_rule.go` + +**Interfaces:** +- Produces one bounded fallback stage from actual model to requested model before first response output. +- Consumes `RelayInfo` routing fields, existing `BillingSettler.Refund`, `SetupContextForSelectedChannel`, and `CacheGetRandomSatisfiedChannel`. + +- [ ] **Step 1: Write failing fallback and billing tests** + +```go +func TestRelayFallsBackToRequestedModelBeforeFirstByte(t *testing.T) { + harness := newRelayHarness(t). + WithRoute("gpt-5-mini", "deepseek-v3"). + WithUpstreamFailure("deepseek-v3", http.StatusServiceUnavailable). + WithUpstreamSuccess("gpt-5-mini", usage(100, 20)) + + response := harness.DoRequest() + + require.Equal(t, http.StatusOK, response.StatusCode) + assert.Equal(t, []string{"deepseek-v3", "gpt-5-mini"}, harness.AttemptedModels()) + assert.Equal(t, 1, harness.InitialBillingRefunds()) + assert.Equal(t, 1, harness.FallbackBillingSettlements()) +} +``` + +Add cases proving no fallback after first byte, no fallback in strict mode, no fallback when disabled, only one cross-model fallback, actual attempted cost retained for administrator audit, and insufficient quota for the requested-model fallback returns an error without a second upstream call. + +- [ ] **Step 2: Run focused controller and billing tests and verify failure** + +Run: `go test ./controller ./service -run 'TestRelayFallsBack|TestBillingSessionRoutingFallback' -count=1` + +Expected: FAIL because the relay loop only retries one model. + +- [ ] **Step 3: Refactor relay attempts into two explicit stages** + +```go +attemptModels := []string{relayInfo.ActualModelName} +if relayInfo.RoutingMode == "optimize_cost" && relayInfo.AllowRoutingFallback && + relayInfo.RequestedModelName != relayInfo.ActualModelName { + attemptModels = append(attemptModels, relayInfo.RequestedModelName) +} +``` + +Each model stage owns its own channel retries and billing session. Before entering stage two, verify no response has been emitted, refund or settle stage one, reset request body and request model, rebuild `RelayInfo` pricing state, then pre-consume the requested model. Persist a `RoutingDecision` after completion with `FallbackTriggered`, final channel, estimated costs, and non-content request features. + +- [ ] **Step 4: Run controller, service, and relay test suites** + +Run: `go test ./controller ./service ./relay/... -run 'Relay|Billing|Routing' -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit bounded fallback** + +```bash +git add controller/relay.go controller/relay_cost_routing_test.go service/billing_session.go service/billing_session_test.go model/routing_rule.go +git commit -m "feat: fall back routed requests safely" +``` + +### Task 7: Disclose Routing in Headers and Compatible Response Extensions + +**Files:** +- Create: `service/routing_response.go` +- Create: `service/routing_response_test.go` +- Modify: `controller/relay.go` +- Modify: `relay/compatible_handler.go` +- Modify: `relay/responses_handler.go` + +**Interfaces:** +- Produces `service.SetRoutingResponseHeaders(*gin.Context, *relaycommon.RelayInfo)` and `service.RoutingResponseMetadata(*relaycommon.RelayInfo) map[string]any`. +- Consumes routing identity from Task 5. + +- [ ] **Step 1: Write failing disclosure tests** + +```go +func TestSetRoutingResponseHeadersDisclosesSubstitution(t *testing.T) { + c, recorder := testContext(t) + info := &relaycommon.RelayInfo{ + RequestedModelName: "gpt-5-mini", ActualModelName: "deepseek-v3", + RoutingMode: "optimize_cost", RoutingRuleID: 9, EstimatedSaving: 0.0062, + } + + SetRoutingResponseHeaders(c, info) + + assert.Equal(t, "deepseek-v3", recorder.Header().Get("X-Nailong-Actual-Model")) + assert.Equal(t, "9", recorder.Header().Get("X-Nailong-Routing-Rule")) + assert.Equal(t, "0.006200", recorder.Header().Get("X-Nailong-Estimated-Saving")) +} +``` + +Add tests that strict requests emit no routing headers and that estimated saving is clamped to zero when non-finite or negative. + +- [ ] **Step 2: Run disclosure tests and verify failure** + +Run: `go test ./service -run TestSetRoutingResponseHeaders -count=1` + +Expected: FAIL because response helpers are undefined. + +- [ ] **Step 3: Implement headers before upstream body forwarding** + +Set headers after the route decision and before any handler can write response bytes. Add the `routing` object only on response formats whose DTO extension is backward compatible; do not rewrite opaque pass-through JSON or SSE chunks in this task. Document that SDKs can always read the response headers and activity API. + +- [ ] **Step 4: Run response and relay tests** + +Run: `go test ./service ./controller ./relay/... -run 'RoutingResponse|Compatible|Responses|Relay' -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit response disclosure** + +```bash +git add service/routing_response.go service/routing_response_test.go controller/relay.go relay/compatible_handler.go relay/responses_handler.go +git commit -m "feat: disclose optimized routing results" +``` + +### Task 8: Add Administrator Rule CRUD and Dry-Run APIs + +**Files:** +- Create: `controller/routing_rule.go` +- Create: `controller/routing_rule_test.go` +- Modify: `router/api-router.go` +- Modify: `controller/channel_authz.go` +- Modify: `i18n/en.yaml` +- Modify: `i18n/zh.yaml` + +**Interfaces:** +- Produces endpoints `GET /api/routing-rules`, `POST /api/routing-rules`, `PUT /api/routing-rules/:id`, `DELETE /api/routing-rules/:id`, and `POST /api/routing-rules/dry-run`. +- Consumes `model.RoutingRule`, `costrouter.Router.Route`, existing administrator authentication and audit conventions. + +- [ ] **Step 1: Write failing authorization, validation, and dry-run tests** + +```go +func TestRoutingRuleDryRunReturnsDeterministicDecision(t *testing.T) { + router := adminRouter(t) + seedRoutingRule(t, ruleFor("gpt-*", "deepseek-v3")) + + response := performAdminJSON(t, router, http.MethodPost, "/api/routing-rules/dry-run", map[string]any{ + "requested_model": "gpt-5-mini", "endpoint": "chat.completions", + "prompt_tokens": 1000, "estimated_output_tokens": 200, + }) + + require.Equal(t, http.StatusOK, response.Code) + assert.JSONEq(t, `{"success":true,"data":{"requested_model":"gpt-5-mini","actual_model":"deepseek-v3","rule_id":1,"substituted":true}}`, response.Body.String()) +} +``` + +Add tests for non-admin denial, invalid patterns, invalid JSON fields, empty candidates, duplicate models, invalid costs, missing rule, audit logging, and deterministic list ordering. + +- [ ] **Step 2: Run controller tests and verify failure** + +Run: `go test ./controller -run TestRoutingRule -count=1` + +Expected: FAIL because endpoints and handlers do not exist. + +- [ ] **Step 3: Implement CRUD, validation, and dry-run handlers** + +Follow existing controller response envelopes and admin audit middleware. Return sanitized rule structures; never return channel keys or unrelated provider configuration. A dry run uses supplied non-content features and current availability but never calls an upstream model or creates a billing session. + +- [ ] **Step 4: Run controller and router tests** + +Run: `go test ./controller ./router -run 'RoutingRule|AdminAudit' -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit management APIs** + +```bash +git add controller/routing_rule.go controller/routing_rule_test.go router/api-router.go controller/channel_authz.go i18n/en.yaml i18n/zh.yaml +git commit -m "feat: manage cost routing rules" +``` + +### Task 9: Add Focused User Routing Controls + +**Files:** +- Modify: `web/src/features/keys/types.ts` +- Modify: `web/src/features/keys/lib/api-key-form.ts` +- Modify: `web/src/features/keys/components/api-keys-mutate-drawer.tsx` +- Test: `web/src/features/keys/lib/__tests__/api-key-form.test.ts` +- Test: `web/src/features/keys/components/__tests__/api-keys-mutate-drawer.test.tsx` +- Modify: `web/src/i18n/locales/en.json` +- Modify: `web/src/i18n/locales/zh.json` +- Modify: all other locale files through the project `i18n-translate` workflow + +**Interfaces:** +- Produces API-key fields `routing_mode`, `routing_max_cost`, `routing_fallback`, and `routing_disclosure_accepted` in existing token create/update requests. +- Consumes token API changes from Task 3. + +- [ ] **Step 1: Extend the failing schema and drawer interaction tests** + +Add a schema test to `web/src/features/keys/lib/__tests__/api-key-form.test.ts` that rejects `optimize_cost` until disclosure is accepted. Add a drawer test to `web/src/features/keys/components/__tests__/api-keys-mutate-drawer.test.tsx` that selects cost optimization, enters `0.02`, enables fallback, accepts disclosure, and asserts the submitted payload below. + +Expected payload: + +```json +{ + "routing_mode": "optimize_cost", + "routing_max_cost": 0.02, + "routing_fallback": true, + "routing_disclosure_accepted": true +} +``` + +- [ ] **Step 2: Run the focused frontend tests and verify failure** + +Run: `cd web; bun test src/features/keys/lib/__tests__/api-key-form.test.ts src/features/keys/components/__tests__/api-keys-mutate-drawer.test.tsx` + +Expected: FAIL because routing controls are not rendered. + +- [ ] **Step 3: Implement typed form fields and accessible controls** + +Extend the Zod schema, `ApiKey` types, defaults, edit hydration, and payload transform with `routing_mode`, `routing_max_cost`, `routing_fallback`, and `routing_disclosure_accepted`. Use the existing drawer component system. Default to strict mode. Display this exact Chinese disclosure beside the opt-in control: `开启后，实际执行模型可能与所选模型不同；系统会显示实际模型，并按实际模型计算额度。` Do not hide or pre-check acceptance. + +- [ ] **Step 4: Complete all locale files with the i18n workflow** + +Use the `i18n-translate` skill to add natural translations for every new literal key to `en`, `zh`, `zh-TW`, `fr`, `ja`, `ru`, and `vi`, then run `cd web; bun run i18n:sync` and verify it reports no missing keys. + +- [ ] **Step 5: Run frontend tests, typecheck, lint, and build** + +Run: `cd web; bun test src/features/keys/lib/__tests__/api-key-form.test.ts src/features/keys/components/__tests__/api-keys-mutate-drawer.test.tsx; bun run typecheck; bun run lint; bun run build` + +Expected: all commands PASS with no missing translation keys. + +- [ ] **Step 6: Commit user controls** + +```bash +git add web/src +git commit -m "feat: add token cost optimization controls" +``` + +### Task 10: Verify the Core Slice and Document Its Operational Contract + +**Files:** +- Create: `docs/nailong-cost-routing.md` +- Modify: `.env.example` +- Modify: `docker-compose.yml` + +**Interfaces:** +- Documents administrator rule configuration, API-key opt-in, request overrides, response headers, fallback limits, privacy behavior, and rollback. +- Consumes all earlier tasks. + +- [ ] **Step 1: Add a black-box compatibility test script or Go test using two local stub upstreams** + +Cover strict routing, optimized substitution, unavailable replacement fallback, response disclosure, exact billing model, and disabled-rule behavior without contacting paid providers. + +- [ ] **Step 2: Run full backend verification** + +Run: `go test ./...` + +Expected: PASS. + +- [ ] **Step 3: Verify independent relaykit build** + +Run: `cd relaykit; $env:GOWORK='off'; go build ./...` + +Expected: PASS. + +- [ ] **Step 4: Run frontend verification** + +Run: `cd web; bun run i18n:sync; bun run build` + +Expected: PASS. + +- [ ] **Step 5: Run database migration smoke tests** + +Run the repository's SQLite migration tests locally and the existing MySQL/PostgreSQL migration jobs in CI. Expected: both new tables and token columns migrate without dialect-specific SQL or repeated schema alterations. + +- [ ] **Step 6: Write operator documentation and rollback steps** + +Document that setting every token to `strict` and disabling all routing rules immediately restores baseline model behavior without removing tables. Document response headers, audit locations, maximum retry stages, and the fact that prompts are not persisted by the router. + +- [ ] **Step 7: Commit verification and documentation** + +```bash +git add docs/nailong-cost-routing.md .env.example docker-compose.yml +git commit -m "docs: add cost routing operations guide" +``` + +## Completion Gate + +Before declaring this plan complete: + +- Every task commit exists and `git status --short` is clean. +- `go test ./...` passes. +- `relaykit` builds with `GOWORK=off`. +- Frontend i18n synchronization and production build pass. +- A strict-mode request is byte-for-byte compatible at the API contract level with the upstream New API baseline, excluding nondeterministic identifiers and timestamps. +- An optimized request can be traced from requested model through actual model and upstream mapping to final billing and response disclosure. +- No test or log fixture contains real API keys or full user prompts. diff --git a/docs/superpowers/plans/2026-08-20-intelligent-routing-policy-control.md b/docs/superpowers/plans/2026-08-20-intelligent-routing-policy-control.md new file mode 100644 index 000000000000..714af7f203d1 --- /dev/null +++ b/docs/superpowers/plans/2026-08-20-intelligent-routing-policy-control.md @@ -0,0 +1,543 @@ +# Intelligent Routing Policy Control Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Deliver durable intelligent-routing policy versions, deterministic administrator rollouts, publication and rollback, root-only management APIs, and request-path rollout resolution. + +**Architecture:** GORM repositories persist immutable published policies and a revisioned singleton rollout. A service layer validates policy documents, performs transactional publication and rollback, and exposes an atomic local runtime snapshot; the request path resolves group targeting and stable traffic buckets without database access. + +**Tech Stack:** Go 1.22+, Gin, GORM v2, testify, existing `common` JSON wrappers, SQLite/MySQL/PostgreSQL-compatible schema. + +**Spec:** `docs/superpowers/specs/2026-08-20-multi-instance-admin-intelligent-routing-design.md` + +## Global Constraints + +- Preserve SQLite, MySQL 5.7.8+, and PostgreSQL 9.6+ compatibility. +- Use `common.Marshal`, `common.Unmarshal`, `common.UnmarshalJsonStr`, or `common.DecodeJson`; do not call `encoding/json` marshal/unmarshal functions. +- Use `lockForUpdate(tx)` for standard GORM row locks. +- Published policies are immutable; rollback creates a new published version. +- All mutation endpoints require root authorization, optimistic concurrency where applicable, and operation audit. +- Do not place user IDs, token IDs, session IDs, prompts, credentials, or arbitrary errors into metric dimensions. +- Keep `relaykit/` independently buildable with `GOWORK=off`. +- Follow TDD: observe each focused test fail before implementing its production behavior. + +## File Structure + +- Create `model/intelligent_routing_policy.go`: durable policy and rollout models plus transaction-safe repository operations. +- Create `model/intelligent_routing_policy_test.go`: SQLite-backed repository contract tests. +- Modify `model/main.go`: include both models in normal and fast migrations. +- Create `service/intelligent_routing/policy_document.go`: canonical document parsing, validation, checksum, and structured validation errors. +- Create `service/intelligent_routing/policy_document_test.go`: deterministic validation and checksum tests. +- Create `service/intelligent_routing/policy_control.go`: draft, publish, rollback, rollout update, and immutable snapshot orchestration. +- Create `service/intelligent_routing/policy_control_test.go`: service transaction, immutability, and conflict tests. +- Create `service/intelligent_routing/rollout.go`: deterministic group matching and stable bucket resolution. +- Create `service/intelligent_routing/rollout_test.go`: rollout resolution behavior tests. +- Create `dto/intelligent_routing.go`: explicit administrator request and response DTOs. +- Create `controller/intelligent_routing.go`: root administrator policy and rollout handlers. +- Create `controller/intelligent_routing_test.go`: handler status, DTO, conflict, and audit tests. +- Modify `controller/audit.go`: register stable intelligent-routing audit templates. +- Modify `router/api-router.go`: register the root-only route group. +- Modify `controller/relay.go`: replace the global-only enable decision with the immutable rollout snapshot while preserving legacy behavior when no durable rollout exists. +- Modify `controller/intelligent_routing_shadow_test.go`: cover scoped shadow/live activation in the relay flow. +- Modify `docs/intelligent-routing-shadow-rollout.md`: document durable policy and rollout administration. + +--- + +### Task 1: Durable Policy and Rollout Models + +**Files:** +- Create: `model/intelligent_routing_policy.go` +- Create: `model/intelligent_routing_policy_test.go` +- Modify: `model/main.go` + +**Interfaces:** +- Produces: `IntelligentRoutingPolicy`, `IntelligentRoutingRollout`, `CreateIntelligentRoutingDraft`, `UpdateIntelligentRoutingDraft`, `ListIntelligentRoutingPolicies`, `GetIntelligentRoutingPolicy`, `GetActiveIntelligentRoutingPolicy`, `PublishIntelligentRoutingPolicy`, `RollbackIntelligentRoutingPolicy`, `GetIntelligentRoutingRollout`, `UpdateIntelligentRoutingRollout`. +- Consumes: global `model.DB`, `lockForUpdate(tx)`, GORM transactions. + +- [ ] **Step 1: Write failing migration and draft repository tests** + +Create a SQLite fixture that assigns `model.DB`, migrates the two new models, creates a draft, fetches it, and rejects updating a non-draft row. Use exact assertions: + +```go +require.NoError(t, DB.AutoMigrate(&IntelligentRoutingPolicy{}, &IntelligentRoutingRollout{})) +draft, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{Status: IntelligentRoutingPolicyDraft, Config: `{"enabled":false}`, Checksum: "sum", CreatedBy: 11}) +require.NoError(t, err) +assert.Equal(t, IntelligentRoutingPolicyDraft, draft.Status) +stored, err := GetIntelligentRoutingPolicy(draft.Id) +require.NoError(t, err) +assert.Equal(t, draft.Id, stored.Id) +``` + +- [ ] **Step 2: Run the focused test and verify it fails** + +Run: `go test ./model -run 'TestIntelligentRoutingPolicy' -count=1` + +Expected: FAIL because the policy and rollout types and repository functions do not exist. + +- [ ] **Step 3: Implement portable models and draft operations** + +Define statuses as string constants and use portable fields: + +```go +type IntelligentRoutingPolicy struct { + Id int64 `json:"id" gorm:"primaryKey"` + Version int `json:"version" gorm:"index"` + Status string `json:"status" gorm:"type:varchar(16);index"` + Config string `json:"config" gorm:"type:text"` + Checksum string `json:"checksum" gorm:"type:varchar(64)"` + SourceVersion int `json:"source_version"` + ChangeNote string `json:"change_note" gorm:"type:varchar(500)"` + CreatedBy int `json:"created_by"` + PublishedBy int `json:"published_by"` + PublishedAt *time.Time `json:"published_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type IntelligentRoutingRollout struct { + Id int64 `json:"id" gorm:"primaryKey"` + Revision int64 `json:"revision"` + PolicyVersion int `json:"policy_version"` + Enabled bool `json:"enabled"` + Mode string `json:"mode" gorm:"type:varchar(16)"` + TrafficPercent int `json:"traffic_percent"` + UserGroups string `json:"user_groups" gorm:"type:text"` + TokenGroups string `json:"token_groups" gorm:"type:text"` + UpdatedBy int `json:"updated_by"` + StartedAt *time.Time `json:"started_at"` + EndedAt *time.Time `json:"ended_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} +``` + +Use GORM `Create`, `First`, `Order`, `Limit`, and conditional `Updates`; return named sentinel errors for not found, immutable policy, and revision conflict. + +- [ ] **Step 4: Add transactional publication, rollback, and rollout conflict tests** + +Assert publication assigns version 1, archives an existing active row when version 2 is published, rollback of version 1 creates active version 3 with `SourceVersion == 1`, and rollout update with the wrong revision returns `ErrIntelligentRoutingRevisionConflict` without changing the stored row. + +- [ ] **Step 5: Implement transactional publication and rollout updates** + +Inside `DB.Transaction`, lock the current active row and latest version query through `lockForUpdate(tx)`, calculate the next version, archive the active row, then update the selected draft. Implement rollback by copying configuration and checksum into a new active row. Implement rollout compare-and-swap with: + +```go +result := tx.Model(&IntelligentRoutingRollout{}). + Where("id = ? AND revision = ?", current.Id, expectedRevision). + Updates(map[string]any{"revision": expectedRevision + 1, /* normalized fields */}) +if result.Error != nil { return result.Error } +if result.RowsAffected != 1 { return ErrIntelligentRoutingRevisionConflict } +``` + +- [ ] **Step 6: Register both tables in normal and fast migrations** + +Add `&IntelligentRoutingPolicy{}` and `&IntelligentRoutingRollout{}` to the existing `AutoMigrate` lists in `model/main.go`; do not add dialect-specific SQL. + +- [ ] **Step 7: Run model tests** + +Run: `go test ./model -run 'TestIntelligentRoutingPolicy|TestIntelligentRoutingRollout' -count=1` + +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add model/intelligent_routing_policy.go model/intelligent_routing_policy_test.go model/main.go +git commit -m "feat: persist intelligent routing policies" +``` + +### Task 2: Policy Document Validation and Canonical Checksum + +**Files:** +- Create: `service/intelligent_routing/policy_document.go` +- Create: `service/intelligent_routing/policy_document_test.go` +- Modify: `setting/intelligent_routing_setting/config.go` +- Modify: `setting/intelligent_routing_setting/config_test.go` + +**Interfaces:** +- Consumes: `intelligent_routing_setting.Config`, `intelligent_routing_setting.Normalize`, billing model configuration accessors. +- Produces: `ValidationIssue`, `ValidatedPolicy`, `ValidatePolicyDocument(raw string) (ValidatedPolicy, []ValidationIssue)`, `CanonicalPolicyJSON(config Config) (string, error)`. + +- [ ] **Step 1: Write failing table tests for valid and invalid documents** + +Cover exact field codes for malformed JSON, oversized JSON, duplicate models, negative prices, excessive attempts, excessive durations, unknown capabilities, and a live-capable document with no models. Assert semantically identical JSON produces the same checksum. + +```go +assert.Equal(t, ValidationIssue{Code: "max_attempts.out_of_range", Field: "max_attempts"}, issues[0]) +assert.Equal(t, first.Checksum, second.Checksum) +``` + +- [ ] **Step 2: Run validation tests and verify they fail** + +Run: `go test ./service/intelligent_routing ./setting/intelligent_routing_setting -run 'TestValidatePolicyDocument|TestNormalizeRejectsExcessive' -count=1` + +Expected: FAIL because structured validation and upper bounds are absent. + +- [ ] **Step 3: Add explicit configuration ceilings** + +Add constants for maximum policy bytes, models, attempts, endpoints per model, duration budgets, context limit, price, and cost multiplier. Extend normalization to reject values above those ceilings while preserving current defaults. + +- [ ] **Step 4: Implement canonicalization and structured validation** + +Parse with `common.UnmarshalJsonStr`, normalize, sort copied model policies by model name, sort copied capability slices, marshal with `common.Marshal`, and hash canonical bytes with SHA-256. Map normalization failures to stable field-specific `ValidationIssue` values; do not expose raw parser internals. + +- [ ] **Step 5: Run focused tests** + +Run: `go test ./service/intelligent_routing ./setting/intelligent_routing_setting -run 'TestValidatePolicyDocument|TestCanonicalPolicy|TestNormalize' -count=1` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add service/intelligent_routing/policy_document.go service/intelligent_routing/policy_document_test.go setting/intelligent_routing_setting/config.go setting/intelligent_routing_setting/config_test.go +git commit -m "feat: validate intelligent routing policies" +``` + +### Task 3: Policy Control Service and Immutable Runtime Snapshot + +**Files:** +- Create: `service/intelligent_routing/policy_control.go` +- Create: `service/intelligent_routing/policy_control_test.go` + +**Interfaces:** +- Consumes: Task 1 repository functions and Task 2 `ValidatePolicyDocument`. +- Produces: `PolicyRepository` interface, `PolicyControl`, `RuntimePolicySnapshot`, `NewPolicyControl`, `CreateDraft`, `UpdateDraft`, `Publish`, `Rollback`, `UpdateRollout`, `RefreshSnapshot`, `Snapshot`. + +- [ ] **Step 1: Write failing service tests with a fake repository** + +Verify invalid drafts never reach the repository, publication requires a non-empty trimmed change note, rollback refreshes the snapshot, and a repository revision conflict is preserved as a service conflict. + +```go +control := NewPolicyControl(repo) +_, issues, err := control.CreateDraft(ctx, `{"max_attempts":999}`, 7) +require.NoError(t, err) +require.NotEmpty(t, issues) +assert.Zero(t, repo.createCalls) +``` + +- [ ] **Step 2: Run the service tests and verify they fail** + +Run: `go test ./service/intelligent_routing -run 'TestPolicyControl' -count=1` + +Expected: FAIL because `PolicyControl` is undefined. + +- [ ] **Step 3: Implement the repository interface and service methods** + +Keep database structs out of request-path code. Store the runtime snapshot in `atomic.Pointer[RuntimePolicySnapshot]`; build a fully validated snapshot before swapping it. Return a deep copy from `Snapshot` so callers cannot mutate shared maps or slices. + +- [ ] **Step 4: Add stale snapshot tests** + +Verify a failed refresh leaves the last valid snapshot unchanged and that a disabled rollout produces a valid disabled snapshot rather than a nil pointer. + +- [ ] **Step 5: Implement refresh failure behavior** + +Load rollout and referenced published policy, validate checksum and document, construct the new snapshot, then atomically store it. Return the loading error without changing the old snapshot. + +- [ ] **Step 6: Run focused tests** + +Run: `go test ./service/intelligent_routing -run 'TestPolicyControl|TestRuntimePolicySnapshot' -count=1` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add service/intelligent_routing/policy_control.go service/intelligent_routing/policy_control_test.go +git commit -m "feat: control intelligent routing policy lifecycle" +``` + +### Task 4: Deterministic Rollout Resolution + +**Files:** +- Create: `service/intelligent_routing/rollout.go` +- Create: `service/intelligent_routing/rollout_test.go` + +**Interfaces:** +- Consumes: `RuntimePolicySnapshot` from Task 3. +- Produces: `RolloutSubject`, `RolloutDecision`, `ResolveRollout(snapshot RuntimePolicySnapshot, subject RolloutSubject) RolloutDecision`. + +- [ ] **Step 1: Write failing deterministic-resolution tests** + +Test disabled rollout, nonmatching user group, nonmatching token group, 0%, 100%, stable repeated bucket, changed policy version, and shadow/live mode preservation. + +```go +first := ResolveRollout(snapshot, RolloutSubject{AccountID: 42, TokenID: 9, UserGroup: "default", TokenGroup: "auto"}) +second := ResolveRollout(snapshot, RolloutSubject{AccountID: 42, TokenID: 9, UserGroup: "default", TokenGroup: "auto"}) +assert.Equal(t, first.Bucket, second.Bucket) +assert.Equal(t, first.Selected, second.Selected) +``` + +- [ ] **Step 2: Run rollout tests and verify they fail** + +Run: `go test ./service/intelligent_routing -run 'TestResolveRollout' -count=1` + +Expected: FAIL because rollout resolution is undefined. + +- [ ] **Step 3: Implement allowlist matching and stable bucketing** + +Use HMAC-SHA256 with a deployment salt injected into the snapshot and the exact tuple `policyVersion/accountID/tokenID`. Convert the first eight digest bytes with `binary.BigEndian.Uint64`, then calculate `bucket := int(value % 100)`. Empty allowlists match all subjects; non-empty lists require exact normalized group membership. + +- [ ] **Step 4: Run focused tests** + +Run: `go test ./service/intelligent_routing -run 'TestResolveRollout' -count=1` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add service/intelligent_routing/rollout.go service/intelligent_routing/rollout_test.go +git commit -m "feat: resolve intelligent routing rollouts" +``` + +### Task 5: Administrator Policy and Rollout API + +**Files:** +- Create: `dto/intelligent_routing.go` +- Create: `controller/intelligent_routing.go` +- Create: `controller/intelligent_routing_test.go` +- Modify: `controller/audit.go` +- Modify: `router/api-router.go` + +**Interfaces:** +- Consumes: `PolicyControl` from Task 3 and repository pagination. +- Produces: root-only policy list/get/create/update/validate/publish/rollback and rollout get/update HTTP endpoints. + +- [ ] **Step 1: Define explicit DTO contracts** + +Create request DTOs with typed fields and response DTOs that exclude GORM internals. Use: + +```go +type IntelligentRoutingPublishRequest struct { + ChangeNote string `json:"change_note"` +} + +type IntelligentRoutingRolloutUpdateRequest struct { + Revision int64 `json:"revision"` + PolicyVersion int `json:"policy_version"` + Enabled bool `json:"enabled"` + Mode string `json:"mode"` + TrafficPercent int `json:"traffic_percent"` + UserGroups []string `json:"user_groups"` + TokenGroups []string `json:"token_groups"` +} + +type IntelligentRoutingValidationIssue struct { + Code string `json:"code"` + Field string `json:"field"` + Message string `json:"message"` +} +``` + +- [ ] **Step 2: Write failing router and controller tests** + +Assert unauthenticated and non-root requests are rejected, valid draft creation returns 201, invalid policy returns structured issues with 400, stale rollout revision returns 409, publish returns the assigned version, rollback returns a new version, and each mutation records the expected audit action. + +- [ ] **Step 3: Run controller tests and verify they fail** + +Run: `go test ./controller ./router -run 'TestIntelligentRoutingAdmin|TestIntelligentRoutingRoutes' -count=1` + +Expected: FAIL because routes and handlers are absent. + +- [ ] **Step 4: Implement handlers and status mapping** + +Decode bodies through `common.DecodeJson`, enforce page size and ID bounds, map validation failures to 400, missing records to 404, stale revisions to 409, and dependency failures to 503. Return the repository error only to server logs; client responses use stable messages. + +- [ ] **Step 5: Register audit actions and root-only routes** + +Add the policy and rollout audit templates to `auditContentTemplates`. Register: + +```go +intelligentRoutingRoute := apiRouter.Group("/intelligent-routing") +intelligentRoutingRoute.Use(middleware.RootAuth()) +{ + intelligentRoutingRoute.GET("/policies", controller.ListIntelligentRoutingPolicies) + intelligentRoutingRoute.GET("/policies/:id", controller.GetIntelligentRoutingPolicy) + intelligentRoutingRoute.POST("/policies", controller.CreateIntelligentRoutingPolicy) + intelligentRoutingRoute.PUT("/policies/:id", controller.UpdateIntelligentRoutingPolicy) + intelligentRoutingRoute.POST("/policies/:id/validate", controller.ValidateIntelligentRoutingPolicy) + intelligentRoutingRoute.POST("/policies/:id/publish", controller.PublishIntelligentRoutingPolicy) + intelligentRoutingRoute.POST("/policies/:version/rollback", controller.RollbackIntelligentRoutingPolicy) + intelligentRoutingRoute.GET("/rollout", controller.GetIntelligentRoutingRollout) + intelligentRoutingRoute.PUT("/rollout", controller.UpdateIntelligentRoutingRollout) +} +``` + +- [ ] **Step 6: Run controller and router tests** + +Run: `go test ./controller ./router -run 'TestIntelligentRoutingAdmin|TestIntelligentRoutingRoutes' -count=1` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add dto/intelligent_routing.go controller/intelligent_routing.go controller/intelligent_routing_test.go controller/audit.go router/api-router.go +git commit -m "feat: expose intelligent routing policy admin api" +``` + +### Task 6: Request-Path Rollout Integration + +**Files:** +- Modify: `controller/relay.go` +- Modify: `controller/intelligent_routing_shadow_test.go` +- Modify: `relay/common/relay_info.go` +- Modify: `service/log_info_generate.go` +- Modify: `service/intelligent_routing_audit_test.go` + +**Interfaces:** +- Consumes: `PolicyControl.Snapshot`, `ResolveRollout`, existing planner and execution path. +- Produces: scoped durable rollout activation with `PolicyVersion`, `RolloutRevision`, `RolloutMode`, and `RolloutBucket` in administrator-only audit. + +- [ ] **Step 1: Write failing relay integration tests** + +Add cases proving: disabled rollout uses the legacy selector; excluded group uses the legacy selector; selected shadow rollout plans but does not switch execution; selected live rollout switches execution; the same account/token stays in the same bucket; audit contains policy version and rollout revision. + +- [ ] **Step 2: Run focused relay tests and verify they fail** + +Run: `go test ./controller ./service -run 'Test.*IntelligentRout.*Rollout|TestGenerateTextOtherInfoAddsAdminOnlyIntelligentRoutingAudit' -count=1` + +Expected: FAIL because relay execution does not consult durable rollout state. + +- [ ] **Step 3: Add rollout metadata to relay state** + +Extend `RelayInfo` with integer policy version, rollout revision, bucket, and bounded mode fields. Extend `appendIntelligentRoutingAdminInfo` to include these fields only under `admin_info.intelligent_routing`. + +- [ ] **Step 4: Resolve rollout once before planning** + +Build `RolloutSubject` from authenticated account ID, token ID, user group, and token group already present in request context. Load one immutable snapshot and resolve it once. Use its validated config for planning; do not reread the database or global config during retries. + +When no durable rollout exists, preserve the current global-setting behavior for backward compatibility. When a durable rollout exists but does not select the subject, use the existing selector without shadow planning. + +- [ ] **Step 5: Run focused integration tests** + +Run: `go test ./controller ./service -run 'Test.*IntelligentRout.*Rollout|TestGenerateTextOtherInfoAddsAdminOnlyIntelligentRoutingAudit' -count=1` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add controller/relay.go controller/intelligent_routing_shadow_test.go relay/common/relay_info.go service/log_info_generate.go service/intelligent_routing_audit_test.go +git commit -m "feat: apply scoped intelligent routing rollouts" +``` + +### Task 7: Startup Refresh, Polling, and Documentation + +**Files:** +- Create: `service/intelligent_routing/policy_refresh.go` +- Create: `service/intelligent_routing/policy_refresh_test.go` +- Modify: `model/main.go` +- Modify: `docs/intelligent-routing-shadow-rollout.md` + +**Interfaces:** +- Consumes: `PolicyControl.RefreshSnapshot`. +- Produces: `StartPolicyRefresh(ctx context.Context, control *PolicyControl, interval time.Duration)` and startup initialization after migrations. + +- [ ] **Step 1: Write failing refresh-loop tests** + +Use a fake clock or explicitly driven refresh channel rather than sleeps. Assert startup performs one refresh, a changed rollout revision replaces the snapshot, a failed refresh retains the prior snapshot, and cancellation stops further repository calls. + +- [ ] **Step 2: Run refresh tests and verify they fail** + +Run: `go test ./service/intelligent_routing -run 'TestPolicyRefresh' -count=1` + +Expected: FAIL because the refresh coordinator is absent. + +- [ ] **Step 3: Implement bounded periodic refresh** + +Provide a coordinator whose production entry point uses a ticker and whose test entry point accepts a receive-only trigger channel. Log refresh failures through existing logging, rate-limited by state transition; never clear a valid snapshot on failure. + +- [ ] **Step 4: Initialize after database migration** + +After the policy tables are migrated and database initialization succeeds, load the current snapshot and start the refresh coordinator. A missing rollout is a valid disabled state. Do not make application startup fail merely because no policy has been created. + +- [ ] **Step 5: Update operations documentation** + +Document draft creation, validation, publication, rollout revision conflicts, scoped shadow/live rollout, rollback, compatibility with the legacy global setting, and the administrator API examples using bounded non-secret fixtures. + +- [ ] **Step 6: Run focused tests** + +Run: `go test ./service/intelligent_routing ./model -run 'TestPolicyRefresh|TestIntelligentRoutingPolicy|TestIntelligentRoutingRollout' -count=1` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add service/intelligent_routing/policy_refresh.go service/intelligent_routing/policy_refresh_test.go model/main.go docs/intelligent-routing-shadow-rollout.md +git commit -m "feat: refresh intelligent routing policy snapshots" +``` + +### Task 8: Phase Verification and Rollback Artifacts + +**Files:** +- Create: `verification-intelligent-routing-policy-control/MODIFIED_FILE` +- Create: `verification-intelligent-routing-policy-control/DIFF_FILE` +- Create: `verification-intelligent-routing-policy-control/VERIFICATION.txt` +- Create: `verification-intelligent-routing-policy-control/ROLLBACK.sh` + +**Interfaces:** +- Consumes: all prior tasks. +- Produces: verified phase completion evidence and a tested rollback copy while leaving the working source changed. + +- [ ] **Step 1: Run focused policy-control tests** + +Run: + +```powershell +$env:GOCACHE="$PWD\.gocache" +go test ./setting/intelligent_routing_setting ./service/intelligent_routing ./model ./controller ./service ./router -count=1 +``` + +Expected: all listed packages PASS. + +- [ ] **Step 2: Run full backend verification** + +Run: + +```powershell +go test ./... -count=1 +go vet ./service/intelligent_routing ./controller ./model ./router +Push-Location relaykit +$env:GOWORK="off" +go build ./... +Pop-Location +git diff --check +``` + +Expected: tests PASS, vet and builds exit 0 with no errors, and diff check exits 0. + +- [ ] **Step 3: Create and test the verification artifacts** + +Preserve the original hash, copy the representative modified `service/intelligent_routing/policy_control.go`, save the exact Git diff, write the literal commands, inputs, outputs, and exit statuses into `VERIFICATION.txt`, and make `ROLLBACK.sh` executable. Run rollback against a separate copy and verify its SHA-256 matches the baseline while `MODIFIED_FILE` remains changed. + +- [ ] **Step 4: Reopen every artifact and verify recorded evidence** + +Run: + +```powershell +Get-Content verification-intelligent-routing-policy-control\VERIFICATION.txt -Raw +Get-Content verification-intelligent-routing-policy-control\DIFF_FILE -Raw | Select-Object -First 1 +Get-Content verification-intelligent-routing-policy-control\ROLLBACK.sh -Raw +Get-FileHash verification-intelligent-routing-policy-control\MODIFIED_FILE +``` + +Expected: all four artifacts open successfully and the verification record matches the latest commands. + +- [ ] **Step 5: Commit phase completion** + +```bash +git add verification-intelligent-routing-policy-control +git commit -m "test: verify intelligent routing policy control" +``` + +## Subsequent Plans + +After this plan passes, create separate implementation plans for: + +1. Redis-backed shared health, quality, stickiness, manual isolation, and safe dependency degradation. +2. Actual-cost telemetry, overview/metrics APIs, durable routing events, alerting, simulation, and bounded replay. + +These plans depend on the immutable policy snapshot and administrator API conventions established here. diff --git a/docs/superpowers/plans/2026-08-20-intelligent-routing-shared-runtime.md b/docs/superpowers/plans/2026-08-20-intelligent-routing-shared-runtime.md new file mode 100644 index 000000000000..abc07ceaae33 --- /dev/null +++ b/docs/superpowers/plans/2026-08-20-intelligent-routing-shared-runtime.md @@ -0,0 +1,105 @@ +# Intelligent Routing Shared Runtime Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace process-local intelligent-routing health, quality, and stickiness with Redis-backed shared state, add manual channel isolation, and fail closed to the legacy selector when shared state is unavailable. + +**Architecture:** Define narrow runtime-store interfaces used by planning and relay completion. Redis implementations use namespaced, expiring keys and atomic commands; in-memory implementations remain deterministic test/single-instance fixtures. A shared-state readiness gate prevents live intelligent routing whenever a durable rollout exists but Redis is unhealthy. + +**Tech Stack:** Go, go-redis/v8, miniredis, Gin, testify. + +**Spec:** `docs/superpowers/specs/2026-08-20-multi-instance-admin-intelligent-routing-design.md` + +## Global Constraints + +- Multi-instance live routing requires a healthy Redis client. +- Redis keys use deployment and schema namespaces and never contain prompts, raw sessions, tokens, or credentials. +- State updates are atomic and all keys expire. +- Redis failure immediately returns the request to the existing selector; it never falls back to divergent per-instance state. +- Root-only manual health operations require a reason and operation audit. +- All existing root tests and the independent `relaykit` build must pass. + +--- + +### Task 1: Runtime Store Interfaces and Readiness Gate + +**Files:** +- Create: `service/intelligent_routing/runtime_store.go` +- Create: `service/intelligent_routing/runtime_store_test.go` +- Modify: `controller/relay.go` + +- [ ] Write failing tests for shared-ready, Redis-disabled, nil-client, and failed-ping states. +- [ ] Run `go test ./service/intelligent_routing -run TestSharedRuntime -count=1` and observe failure. +- [ ] Define `HealthStore`, `QualityStore`, and `StickyStore` interfaces plus `SharedRuntime` with an atomic readiness state. +- [ ] Gate durable live rollout selection on `DefaultSharedRuntime.Ready()`; shadow planning may continue without changing execution. +- [ ] Run focused controller and service tests and commit `feat: gate shared intelligent routing runtime`. + +### Task 2: Redis Channel Health and Manual Isolation + +**Files:** +- Create: `service/intelligent_routing/redis_health.go` +- Create: `service/intelligent_routing/redis_health_test.go` +- Modify: `service/intelligent_routing/catalog.go` +- Modify: `controller/relay.go` + +- [ ] Write miniredis tests proving two clients share observations, the 60-second window is pruned, transitions match existing thresholds, and manual isolation overrides automatic state. +- [ ] Run the focused tests and observe missing Redis behavior. +- [ ] Store bounded outcome members in a sorted set, isolation metadata in a hash, and expirations in an atomic pipeline. +- [ ] Replace request-path health reads/writes with the `HealthStore` interface. +- [ ] Run health, catalog, and controller tests and commit `feat: share intelligent routing health state`. + +### Task 3: Redis Quality and Stickiness + +**Files:** +- Create: `service/intelligent_routing/redis_quality.go` +- Create: `service/intelligent_routing/redis_quality_test.go` +- Create: `service/intelligent_routing/redis_stickiness.go` +- Create: `service/intelligent_routing/redis_stickiness_test.go` +- Modify: `controller/relay.go` + +- [ ] Write miniredis tests proving cross-client quality counters, cold-start priors, shared sticky reads, policy mismatch invalidation, expiry, and atomic removal after two validation failures. +- [ ] Run tests and observe failure. +- [ ] Implement expiring hashes and a Lua compare/update script for validation failures. +- [ ] Pass policy version through sticky records and replace request-path local stores with interfaces. +- [ ] Run focused tests and commit `feat: share intelligent routing quality and stickiness`. + +### Task 4: Root Health and Quality Operations API + +**Files:** +- Modify: `dto/intelligent_routing.go` +- Modify: `controller/intelligent_routing.go` +- Modify: `controller/audit.go` +- Modify: `router/api-router.go` +- Create: `controller/intelligent_routing_runtime_test.go` + +- [ ] Write failing tests for health listing, isolate/recover/reset, quality listing/reset, root authorization, required reasons, and audit actions. +- [ ] Implement stable DTO responses and map Redis unavailability to HTTP 503. +- [ ] Register the health and quality routes from the design specification. +- [ ] Run controller/router tests and commit `feat: operate intelligent routing runtime state`. + +### Task 5: Startup Wiring and Failure Recovery + +**Files:** +- Modify: `main.go` +- Create: `service/intelligent_routing/runtime_monitor.go` +- Create: `service/intelligent_routing/runtime_monitor_test.go` +- Modify: `docs/intelligent-routing-shadow-rollout.md` + +- [ ] Write tests using driven ticks: failed ping marks runtime unavailable, successful ping plus policy refresh restores readiness, cancellation stops monitoring, and no live request uses local fallback state. +- [ ] Wire Redis stores only after `InitRedisClient`; keep explicit in-memory mode limited to tests/single-instance configuration. +- [ ] Document dependency degradation, recovery, key namespaces, TTLs, and manual operations. +- [ ] Run focused tests and commit `feat: monitor intelligent routing shared runtime`. + +### Task 6: Verification, Rollback, and PR Update + +**Files:** +- Create: `verification-intelligent-routing-shared-runtime/MODIFIED_FILE` +- Create: `verification-intelligent-routing-shared-runtime/DIFF_FILE` +- Create: `verification-intelligent-routing-shared-runtime/VERIFICATION.txt` +- Create: `verification-intelligent-routing-shared-runtime/ROLLBACK.sh` + +- [ ] Run `go test ./... -count=1`. +- [ ] Run `go vet ./service/intelligent_routing ./controller ./model ./router`. +- [ ] Run `cd relaykit && GOWORK=off go build ./...` and `git diff --check`. +- [ ] Create and execute the four rollback artifacts against a separate copy, reopen them, and record exact hashes, commands, outputs, and exit statuses. +- [ ] Commit `test: verify intelligent routing shared runtime`, push the branch, and update PR #6943 with the new scope and proof. diff --git a/docs/superpowers/specs/2026-08-17-cost-optimized-intelligent-routing-design.md b/docs/superpowers/specs/2026-08-17-cost-optimized-intelligent-routing-design.md new file mode 100644 index 000000000000..e42f9d6c9cb4 --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-cost-optimized-intelligent-routing-design.md @@ -0,0 +1,316 @@ +# Cost-Optimized Intelligent Routing Design + +## 1. Objective + +Add a backend-only intelligent routing layer that keeps the client-selected model identifier stable while selecting an eligible model and provider endpoint for each request. The router minimizes expected total inference cost subject to explicit capability, quality, reliability, latency, retry-budget, and billing-safety constraints. + +The router must not treat the client-selected model as the automatic final fallback. It builds a request-specific candidate graph and can move between provider endpoints and model tiers until the request succeeds or the retry budget is exhausted. + +## 2. Scope + +### Included + +- OpenAI-compatible chat and response relay requests. +- Model and provider capability filtering. +- Four model capability/cost tiers. +- Local deterministic request classification. +- Learned quality prediction after sufficient observations exist. +- Expected-cost ranking. +- Provider health, circuit breaking, retry budgets, and cross-model fallback. +- Conditional session stickiness for prompt-cache savings. +- Deterministic response validation. +- Routing decision, token, cost, and saturation audit data. +- Shadow traffic and progressive rollout for new candidates. + +### Excluded from the first release + +- A general-purpose external router service. +- Per-request online LLM judging for ordinary traffic. +- Automatic model onboarding without benchmark approval. +- Automatic price changes exposed to clients. +- Routing for non-text generation endpoints until their billing and validation boundaries are designed separately. + +## 3. Architectural Boundaries + +The implementation follows the existing Router -> Controller -> Service -> Model layering. + +1. The relay request parser preserves the requested model as the client-facing model identifier. +2. A routing service derives requirements and produces an immutable route plan. +3. Relay execution consumes the plan one attempt at a time. +4. Provider adapters remain responsible only for protocol conversion and upstream execution. +5. The routing service receives endpoint observations without importing provider-specific behavior. +6. Billing uses the actual successful execution node and the existing checked quota-conversion helpers. +7. Logs persist both the requested model and actual execution node for administrative audit. + +The router is not added to `relaykit/` unless its interfaces can remain independent of the root module. Any later public API change in `relaykit/` requires an independent `GOWORK=off go build ./...` verification. + +## 4. Core Data Model + +### Route request + +The routing input contains: + +- Requested model identifier. +- Normalized request modality and parameters. +- Estimated input and maximum output tokens. +- Conversation fingerprint or explicit session identifier. +- Required capabilities. +- Account, group, region, and policy constraints. +- A request-correlated deadline and maximum cost budget. + +### Candidate node + +A candidate node is the tuple `(model, channel, endpoint)` and contains: + +- Model tier and capabilities. +- Context and output limits. +- Input, output, cache-read, and cache-write prices. +- Rolling success, latency, and throughput observations. +- Circuit state and available concurrency. +- Predicted success probability and output length for the current request. +- Expected total cost for the current request. + +### Route plan + +A route plan is immutable after execution starts and contains: + +- The feature snapshot used for the decision. +- Quality threshold and eligible model set. +- Ordered model groups. +- Ordered endpoints inside each model group. +- Maximum attempts, elapsed time, and cost multiplier. +- Validation requirements. +- Explanation codes suitable for administrative audit. + +## 5. Four Model Tiers + +The tier is a capability and quality prior, not a mandatory linear retry sequence. + +| Tier | Purpose | Typical workloads | +|---|---|---| +| L0 | Extraction and transformation at minimum cost | classification, rewriting, short translation, field extraction | +| L1 | Low-cost general work | summaries, routine questions, simple code explanation | +| L2 | Reliable structured and reasoning work | tools, JSON Schema, code generation, long context, multimodal input | +| L3 | Highest-quality eligible candidates | complex reasoning, difficult coding, strict tool use, high-value workloads | + +Each request receives a tailored candidate graph. Ineligible tiers or models are omitted rather than attempted. + +## 6. Routing Algorithm + +### 6.1 Local feature extraction + +Feature extraction performs no model call and targets a P95 latency below 1 ms. It derives token estimates, conversation depth, context utilization, modality, tool and schema requirements, task hints, output constraints, language, streaming mode, and session state. + +### 6.2 Hard capability filtering + +A node is eligible only if it: + +- Supports every required modality and request parameter. +- Has sufficient context and output capacity. +- Supports required tool or structured-output semantics. +- Satisfies region and data-policy constraints. +- Has an available channel and is not in an open circuit. +- Fits the absolute request cost ceiling. + +Hard-filter failures are not retryable attempts. + +### 6.3 Quality prediction + +The cold-start router uses deterministic task rules plus conservative model-tier priors. After sufficient observations, a local predictor estimates: + +`P(answer meets quality threshold | request features, model)` + +Initial learned routing uses embedding nearest neighbors with 30 neighbors and beta smoothing: + +`p_success = (successes + 8) / (samples + 10)` + +A trained model may replace nearest-neighbor estimation only after the dataset has at least 10,000 valid observations, at least 500 observations for each principal model, and at least 200 observations for each supported task category. + +Initial minimum success probabilities are: + +| Task | Minimum probability | +|---|---:| +| Translation, rewriting, summarization | 0.88 | +| General question answering | 0.90 | +| Code generation | 0.93 | +| Information extraction | 0.94 | +| Mathematics and complex reasoning | 0.95 | +| JSON Schema output | 0.97 | +| Tool calling | 0.98 | + +If no candidate meets the threshold, the router selects by descending predicted success probability rather than by cost. + +### 6.4 Expected total cost + +For each eligible model: + +`expected_cost = input_cost + output_cost + cache_cost + retry_risk_cost + router_cost` + +Output cost uses a model-specific output-length estimate. Retry risk cost is the endpoint failure probability multiplied by the expected cost of the next eligible attempt. All quota conversions use the checked helpers in `common/quota_math.go`; clamp information must reach the existing saturation audit path before the consume log is written. + +### 6.5 Model selection + +1. Keep only models whose predicted success probability meets the task threshold. +2. Sort the remaining models by expected total cost. +3. Preserve the highest-success remaining model as the final-attempt candidate. +4. If the qualified set is empty, sort all eligible models by predicted success probability. + +### 6.6 Endpoint selection + +Endpoint sorting uses a lexicographic tuple rather than a blended scalar: + +`(health_tier, expected_total_cost, p95_latency, recent_failure_rate, queue_depth)` + +Health tiers are: + +- `HEALTHY`: rolling 60-second success rate at least 99%. +- `DEGRADED`: rolling success rate from 95% to below 99%. +- `PROBATION`: insufficient samples or newly enabled endpoint. +- `OPEN`: rolling success rate below 95% or a fatal configuration failure. + +Endpoints in `OPEN` state are excluded until their circuit permits a probe. + +## 7. Execution and Fallback Graph + +The normal order is: + +1. Selected model's cheapest healthy endpoint. +2. A second healthy endpoint for the same model. +3. Another qualified model in the same capability tier. +4. A higher-quality eligible model. +5. The preserved highest-success candidate as the final attempt. + +Retryable conditions include connection failure, pre-stream disconnect, timeout, 408, 429, 502, 503, and 504. Authentication failure, exhausted channel balance, unsupported parameters, and context overflow mutate endpoint eligibility instead of repeating the same request unchanged. + +The first release uses these budgets: + +- At most four total upstream attempts. +- At most two endpoint attempts for one model. +- At most 30 seconds accumulated routing time for non-streaming requests. +- At most 12 seconds to first byte for streaming requests. +- At most 2.5 times the first node's expected cost. + +When any budget is reached, the router performs at most one final attempt using the remaining candidate with the highest predicted success probability. + +## 8. Response Validation + +Every successful transport response receives deterministic validation appropriate to the request: + +- Non-empty and not unexpectedly truncated. +- Valid JSON and JSON Schema when requested. +- Valid tool name and arguments. +- Required fields present. +- Requested output language and basic format present. +- No obvious incomplete code fence or malformed structured output. + +Validation failure updates the observation as a model-quality failure and moves to the next compatible node. + +Semantic judging is limited to new-model evaluation, boundary predictions within 0.03 of the quality threshold, explicitly high-value workloads, and sampled quality measurement. Mature models use a 1% sample, newly enabled models use 10%, and their first 200 requests use full evaluation. + +## 9. Session Stickiness and Cache Economics + +The router derives a conversation fingerprint or accepts an explicit session identifier. A successful model and provider remain preferred only while: + +- The node remains healthy. +- It supports the current task. +- Its expected cost is no more than 1.15 times the cheapest qualified alternative. +- The task category has not materially changed. + +The route is recomputed when the task changes, context utilization exceeds 70%, the node degrades, another node offers more than 15% expected savings, or two consecutive validations fail. + +## 10. Observability and Feedback + +Each request records: + +- Requested model. +- Ordered candidate graph and explanation codes. +- Every attempted model, channel, and endpoint. +- Failure or rejection reason for each attempt. +- Predicted quality and expected cost. +- Actual token usage, latency, and charged cost. +- Cache usage and route stickiness. +- Validation outcome and quota-saturation marker. + +Daily routing reports calculate: + +- Cost saving against the requested-model baseline. +- Quality retention against benchmark and sampled judge results. +- First-route success rate. +- Multi-attempt rate. +- Final failure rate. +- Added routing latency. +- Predictor Expected Calibration Error. + +Initial release targets are at least 30% cost saving, at least 95% quality retention, at least 92% first-route success, at most 8% multi-attempt requests, at most 0.5% final failures, no more than 10 ms P95 routing overhead, and predictor ECE no higher than 0.05. + +If quality retention falls below 95%, increase the affected task threshold by 0.02 and reduce low-cost-model traffic. If quality retention remains above 98% while savings are below target, lower the affected threshold by 0.01. Changes require an audit record and are rate-limited to one adjustment per task category per day. + +## 11. Model Onboarding + +New models progress through: + +1. Offline benchmark. +2. Shadow traffic with no client-visible response. +3. 1% live traffic. +4. 5% live traffic. +5. 20% live traffic. +6. Normal learned routing. + +Promotion requires at least 200 samples, a passing quality threshold, final failure rate below 1%, and no severe structured-output or tool-use regression. A model is automatically demoted if its five-minute error rate exceeds 5%, its ten-minute quality falls below threshold, or its P95 latency exceeds twice its established baseline. + +## 12. Configuration and Administrative Controls + +Administrators can: + +- Assign models to tiers and capabilities. +- Configure prices and hard cost ceilings. +- Enable or disable model and channel candidates. +- Set task-specific quality thresholds. +- Inspect route decisions and endpoint health. +- Pin or exclude models for controlled experiments. +- Roll out a routing-policy version gradually. +- Roll back to a prior policy version without changing client configuration. + +Routing policies are versioned. Every route decision stores its policy version so historical behavior remains reproducible. + +## 13. Testing Strategy + +### Unit tests + +- Deterministic feature and requirement extraction. +- Capability filtering for tools, schemas, modalities, and context limits. +- Exact expected-cost calculation and checked quota conversion. +- Threshold behavior and calibrated probability selection. +- Endpoint tuple ordering and circuit transitions. +- Retry-budget and final-attempt invariants. +- Response validation contracts. + +### Integration tests + +- Same-model provider failover. +- Cross-model fallback without forced requested-model recovery. +- Context-overflow rerouting. +- Tool and JSON Schema compatibility filtering. +- Streaming failure before and after response commitment. +- Billing against the actual successful execution node. +- Audit completeness for multi-attempt requests. + +### Evaluation tests + +- Fixed labeled request suite per task category. +- Cost-quality Pareto comparison against always using the requested model. +- Shadow comparisons for newly added candidates. +- Calibration and regression reports by model, task, language, and context band. + +All new Go tests use `require` for setup and fatal assertions and `assert` for non-fatal comparisons. Database fixtures must be explicit and remain compatible with SQLite, MySQL, and PostgreSQL. + +## 14. Rollout + +1. Add observations and route-plan generation with routing disabled. +2. Run shadow decisions and compare them to existing execution. +3. Enable same-model provider optimization. +4. Enable cross-model routing for L0 tasks at 1%. +5. Expand L0, then L1, L2, and L3 independently after their metrics pass. +6. Enable learned routing only after the observation thresholds are met. +7. Preserve an immediate configuration rollback to the previous policy version. + diff --git a/docs/superpowers/specs/2026-08-17-nailong-cost-routing-design.md b/docs/superpowers/specs/2026-08-17-nailong-cost-routing-design.md new file mode 100644 index 000000000000..479e9f3a55a6 --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-nailong-cost-routing-design.md @@ -0,0 +1,559 @@ +# 奶龙工作室成本优化路由二次开发设计 + +## 1. 文档目的 + +本文定义基于 New API 二次开发的“奶龙工作室”内测平台架构。平台面向不超过 100 名受邀用户，保留 New API 当前支持的全部上游渠道，并增加用户主动开启的跨模型成本优化能力。 + +首版不接入支付。系统向用户发放测试额度，同时完整记录平台实际承担的上游成本，为后续商业化保留可靠的账务基础。 + +## 2. 产品目标与约束 + +### 2.1 产品目标 + +- 提供兼容 OpenAI 等现有协议的统一 API 入口。 +- 用户可以选择目标模型，并自主开启“省钱优化”。 +- 开启优化后，系统可以选择满足请求能力要求的更低成本模型。 +- 替代模型不可用时，优先切换同模型渠道，再回退到用户原选模型。 +- 用户可以查看实际执行模型、估算节省和请求明细。 +- 管理员可以配置、模拟和审计路由规则。 +- 关闭省钱优化时，现有 New API 行为与渠道兼容性不得退化。 + +### 2.2 约束 + +- 首版规模为 100 人以内的邀请制内测。 +- 首版保留 New API 已支持的全部渠道。 +- 首版不接支付、代理分销、返佣或自动充值。 +- 首版使用确定性规则，不调用额外模型分析用户提示词。 +- 跨模型替代必须由用户明确开启。 +- 产品界面可增加“奶龙工作室”品牌，但保留 New API、QuantumNous、许可证及必要归属信息。 +- 公开发布前，由运营方确认“奶龙”名称和视觉素材的商标、著作权及其他使用授权。 + +## 3. 方案选择 + +采用 New API 单体内嵌成本路由方案。在渠道分发之前新增独立 `CostRouter` 服务，由它决定实际执行模型，然后复用 New API 已有的渠道选择、协议适配、重试、预扣、结算和日志链路。 + +不采用以下方案： + +- 不直接扩展渠道 `model_mapping` 承担动态路由。该能力适合静态上游名称转换，无法清晰表达用户请求模型、实际模型和上游模型三层语义。 +- 不在首版拆分独立路由微服务。对于 100 人内测，它会增加部署、网络和故障处理复杂度，收益不足。 + +## 4. 总体架构 + +```text +用户 / OpenAI SDK + │ + ▼ +身份认证与 Token 权限校验 + │ + ▼ +成本优化路由 CostRouter + ├─ 解析有效路由模式 + ├─ 保存 requested_model + ├─ 匹配并校验替代规则 + └─ 生成 actual_model 与回退计划 + │ + ▼ +New API Distributor + ├─ 根据 actual_model 查找渠道 + ├─ 校验请求路径与能力 + └─ 按成本和健康状态选择渠道 + │ + ▼ +Token 估算、价格快照与额度预扣 + │ + ▼ +New API Relay / Provider Adapter + │ + ▼ +上游模型服务 + │ + ▼ +流式或非流式响应 + │ + ▼ +实际用量结算、日志与性能指标 +``` + +首版继续采用 New API 的 Go、Gin、GORM、React 技术栈。应用保持无状态，运行时配置和规则缓存在 Redis 与进程内存中，持久数据保存在 PostgreSQL。新增数据库代码仍须兼容 SQLite、MySQL 和 PostgreSQL。 + +## 5. 核心领域语义 + +每次请求必须区分三个模型名称： + +| 字段 | 含义 | 主要用途 | +|---|---|---| +| `requested_model` | 用户选择并提交的模型 | 用户授权、页面展示、用量主视图 | +| `actual_model` | 成本路由实际选择的模型 | 能力校验、定价、预扣、结算、渠道选择 | +| `upstream_model` | 渠道映射后发送给供应商的名称 | Provider Adapter 与上游协议 | + +三者不得互相覆盖。Gin Context 和 `RelayInfo` 需要提供明确字段，避免继续让 `OriginalModel` 同时承载多种语义。 + +新增请求上下文： + +- `requested_model` +- `actual_model` +- `routing_mode` +- `routing_rule_id` +- `routing_reason` +- `routing_fallback_model` + +## 6. CostRouter 设计 + +### 6.1 职责 + +`CostRouter` 只负责根据请求和配置选择实际模型，不负责选择渠道、请求上游或扣减额度。 + +输入： + +- 用户选择的模型 +- 请求端点与能力特征 +- 用户、API Key 和单次请求的路由设置 +- 用户允许访问的模型范围 +- 当前启用的路由规则与模型能力元数据 + +输出： + +- 实际执行模型 +- 命中的规则 ID +- 替代理由 +- 是否发生替代 +- 失败时的回退模型 + +### 6.2 路由模式 + +- `strict`：严格使用用户选择的模型，只允许在同模型的不同渠道之间切换。 +- `optimize_cost`：允许使用管理员配置的低成本替代模型，并在替代链路失败后回退用户原选模型。 + +有效配置优先级： + +```text +单次请求设置 > API Key 默认设置 > 用户默认设置 > 系统默认 strict +``` + +### 6.3 请求处理顺序 + +1. 完成 Token 鉴权。 +2. 检查用户是否有权请求 `requested_model`。 +3. 解析有效路由模式。 +4. `strict` 模式直接令 `actual_model = requested_model`。 +5. `optimize_cost` 模式匹配路由规则并生成候选模型。 +6. 按端点、能力、上下文、价格、渠道状态和用户权限过滤候选模型。 +7. 选择预计总成本最低的合格模型作为 `actual_model`。 +8. 将供下游解析的请求体模型设置为 `actual_model`，同时保留原始请求模型。 +9. Distributor 根据 `actual_model` 选择渠道。 +10. 按 `actual_model` 的价格快照预扣与结算。 + +### 6.4 规则定义 + +每条规则包含： + +- 名称、状态与优先级 +- 原始模型精确模式或通配符模式 +- 有序的替代模型列表 +- 允许的请求端点 +- 必须具备的能力 +- 最大上下文长度 +- 适用用户组 +- 单次请求最高预计成本 +- 是否允许回退原模型 +- 生效时间 + +替代模型按规则优先级产生候选集，但最终必须通过能力与成本筛选。首版不允许管理员提交任意脚本或表达式作为路由逻辑。 + +### 6.5 能力过滤 + +以下任一条件不满足时排除候选模型： + +- 不支持当前 API 端点。 +- 不支持请求所需的文本、图片、音频、工具调用、结构化输出或其他能力。 +- 上下文窗口不足。 +- 没有可用渠道。 +- 用户或 API Key 无权使用实际模型。 +- 模型或全部可用渠道处于熔断状态。 +- 预计价格超过用户设置的成本上限。 + +原始模型和实际模型必须同时通过访问授权。跨模型替代不得成为绕过 Token 模型限制的路径。 + +### 6.6 成本估算 + +```text +预计成本 = +输入 Token × 输入单价 ++ 预计输出 Token × 输出单价 ++ 请求固定费用 +``` + +预计输出 Token 按以下来源依次确定： + +1. 使用请求中的输出上限作为安全边界，而非直接视为预计实际用量。 +2. 使用同模型、同端点的历史输出/输入比例。 +3. 历史数据不足时使用管理员配置的默认比例。 +4. 结果受模型最大输出长度和全局安全上限约束。 + +历史统计采用中位数或截尾平均值，降低极端请求对估算的影响。所有价格和额度计算复用 New API 的安全换算方法，禁止裸浮点到整数转换导致负数、溢出或异常信用。 + +## 7. 渠道选择、重试与熔断 + +### 7.1 渠道动态评分 + +同一模型的候选渠道综合以下因素： + +- 价格：50% +- 成功率：25% +- 首 Token 和总响应延迟：15% +- 限流与剩余额度健康度：10% + +权重作为首版默认值，由系统配置管理。选择流程先排除不健康渠道，再从最低成本的一组健康渠道中按权重分配流量，避免流量集中到单一上游。 + +### 7.2 熔断 + +渠道出现连续上游错误、短窗口高错误率、持续 429、健康检查失败或管理员手动暂停时进入熔断。默认退避阶段为 30 秒、2 分钟和 10 分钟。恢复时先放入少量探测流量，连续成功后恢复正常权重。 + +### 7.3 回退顺序 + +```text +actual_model 的其他健康渠道 + ↓ 全部失败 +requested_model 的健康渠道 + ↓ 全部失败 +返回错误 +``` + +只有在首个响应字节发出前才允许切换渠道或模型。流式响应已经开始后不得替换模型，以免产生重复内容、语义断裂和重复计费。 + +每个客户端请求复用同一个 `request_id`，每次上游尝试使用独立 `attempt_id`。首版最多包含一个替代模型阶段和一个原模型回退阶段，不进行无限跨模型重试。 + +## 8. 数据模型 + +复杂配置以 `TEXT` 保存 JSON，并在 Go 服务层统一解析和校验，避免依赖数据库特定 JSON 操作。 + +### 8.1 `routing_rules` + +保存跨模型替代规则： + +- `id` +- `name` +- `status` +- `priority` +- `requested_model_pattern` +- `replacement_models_json` +- `endpoints_json` +- `capabilities_json` +- `max_context_tokens` +- `max_estimated_cost` +- `fallback_to_requested` +- `user_groups_json` +- `effective_from` +- `effective_until` +- `created_at` +- `updated_at` + +### 8.2 `routing_preferences` + +保存用户或 API Key 偏好： + +- `id` +- `user_id` +- `token_id` +- `mode` +- `max_cost_per_request` +- `allow_fallback` +- `disclosure_version` +- `disclosure_accepted_at` +- `created_at` +- `updated_at` + +`token_id` 有值时表示 API Key 级设置，否则表示用户默认设置。 + +### 8.3 `routing_decisions` + +保存每次路由决策： + +- `id` +- `request_id` +- `user_id` +- `token_id` +- `requested_model` +- `actual_model` +- `routing_mode` +- `rule_id` +- `decision_reason` +- `fallback_model` +- `fallback_triggered` +- `estimated_original_cost` +- `estimated_actual_cost` +- `actual_cost` +- `channel_id` +- `request_features_json` +- `created_at` + +该表不保存完整提示词。`request_features_json` 仅包含上下文长度、能力标志、端点等非内容元数据。 + +### 8.4 渠道指标 + +优先复用 New API 现有 `perf_metric`。如果现有字段不足，则扩展短时间窗口指标，以记录渠道与模型维度的请求量、成功量、429、5xx、首 Token 延迟、总延迟、输出 Token 和生成时长。不得建立与现有性能指标含义重复的新表。 + +## 9. 计费与账务一致性 + +成本路由必须在预扣之前完成。计费流程为： + +```text +选定 actual_model + → 获取并固定价格快照 + → 估算 Token + → 预扣额度 + → 调用上游 + → 获取实际用量 + → 按价格快照结算 + → 多退少补 +``` + +价格快照在请求开始时固定，避免请求过程中调价导致预扣与结算使用不同价格。 + +替代模型在输出前失败并回退原模型时： + +1. 结束替代模型的上游尝试并记录可能产生的渠道成本。 +2. 退款或结算替代阶段的用户计费会话。 +3. 获取原模型的新价格快照。 +4. 为原模型重新预扣。 +5. 独立保存尝试成本和最终成功请求成本。 + +首版免费不代表计费为零。用户使用测试额度，平台实际渠道成本单独统计。价格缺失或价格非法时拒绝调用，不得默认免费。 + +## 10. API 契约 + +### 10.1 请求 + +OpenAI 兼容请求保持不变，并允许可选扩展： + +```json +{ + "model": "gpt-5-mini", + "messages": [], + "routing": { + "mode": "optimize_cost", + "max_cost": 0.02, + "fallback": true + } +} +``` + +同时支持请求头： + +```http +X-Nailong-Routing-Mode: optimize_cost +X-Nailong-Max-Cost: 0.02 +``` + +请求设置只能在 API Key 权限允许的范围内收紧或启用已授权能力，不能通过请求头绕过 API Key 或用户限制。 + +### 10.2 响应 + +用户界面、活动日志和用量 API 同时展示请求模型与实际模型。标准响应字段尽量保持兼容，并通过扩展对象返回路由信息： + +```json +{ + "model": "gpt-5-mini", + "routing": { + "mode": "optimize_cost", + "substituted": true, + "actual_model": "deepseek-v3", + "estimated_saving": 0.0062 + } +} +``` + +流式响应在兼容的 usage 尾块中携带路由信息，并通过以下响应头提供最小信息： + +- `X-Nailong-Actual-Model` +- `X-Nailong-Routing-Rule` +- `X-Nailong-Estimated-Saving` + +上述扩展仅在用户已启用成本优化时出现。节省金额属于反事实估算，界面和 API 文档必须明确标记为估算值。 + +## 11. 用户端与管理后台 + +### 11.1 用户端 + +- 省钱优化总开关及机制说明 +- 用户级和 API Key 级路由偏好 +- 单次请求成本上限 +- 是否允许失败后回退原模型 +- 原选模型、实际模型和估算节省明细 +- 累计估算节省 +- 隐私说明和成本优化确认记录 + +模型选择界面仍以用户选择的模型为主。启用优化后必须清晰显示“可能使用更低成本的兼容模型”。 + +### 11.2 管理后台 + +- 路由规则创建、复制、排序、启停和测试 +- 候选模型能力配置 +- 模型与渠道价格管理 +- 渠道实时健康状态 +- 路由模拟器 +- 替代率、回退率、成功率和估算节省 +- 规则效果与用户反馈分析 +- 路由决策和管理员操作审计 + +规则修改写入审计日志，并通过 Redis 发布缓存失效通知。单机和未来多实例部署使用同一缓存接口。 + +## 12. 注册、额度与安全 + +- 使用邀请码控制 100 人以内的内测规模。 +- 邀请码支持次数、有效期和指定用户组。 +- 注册需要邮箱验证，并限制同 IP、设备或邮箱的短时间注册频率。 +- 新用户获得管理员配置的测试额度。 +- API Key 只在创建时展示一次，服务端保存不可逆摘要。 +- 上游渠道密钥使用主密钥加密保存，主密钥通过环境变量注入。 +- 管理员操作、路由规则修改和额度调整写入审计日志。 +- 日志不记录 Authorization Header、上游密钥或完整提示词。 +- 管理接口独立限流，并建议管理员启用 WebAuthn 或双因素认证。 +- 反向代理负责 HTTPS、请求体大小和连接限制。 +- 用户首次启用成本优化时记录说明版本与确认时间。 + +## 13. 异常与降级策略 + +- 请求参数、权限、上下文或能力不匹配返回 4xx，不重试。 +- 没有合格替代模型时使用用户原选模型；原模型也不可用时返回明确错误。 +- 渠道超时、429 或 5xx 只在首字节前重试。 +- Redis 不可用时，规则读取降级到数据库或进程内最后一次有效缓存。 +- 数据库不可用时停止新上游调用，避免产生无法审计或结算的成本。 +- 无法读取价格、完成权限校验或创建计费会话时拒绝请求。 +- 客户端断开时取消上游请求，并结算已经产生的实际用量。 + +## 14. 可观测性与指标 + +记录并展示： + +- 请求成功率、429 和 5xx 比例 +- 首 Token 延迟和总响应延迟 +- 输入、输出及缓存 Token +- 请求模型、实际模型和上游模型 +- 渠道和上游尝试次数 +- 预计原模型成本、实际成本与估算节省 +- 替代率与回退率 +- 规则命中次数 +- 用户点赞与点踩 + +首版通过邮件或 Webhook 对渠道高错误率、上游余额不足、数据库或 Redis 不可用、日成本超限、额度异常消耗和路由回退率异常发出告警。 + +## 15. 测试策略 + +### 15.1 单元测试 + +- `strict` 模式不会跨模型替代。 +- `optimize_cost` 只产生规则允许的候选模型。 +- 端点、能力、上下文、价格和权限过滤正确。 +- 候选模型按预计总成本稳定排序。 +- 规则优先级和冲突解决确定。 +- 缺失或非法价格安全失败。 +- 额度换算不会产生负数、溢出或异常信用。 +- 三层模型名称不会互相覆盖。 +- 首字节输出后禁止模型回退。 + +### 15.2 集成测试 + +- Chat Completions、Responses、Claude Messages、Gemini 原生接口及现有其他渠道。 +- 流式与非流式响应。 +- 替代成功、同模型渠道重试和原模型回退。 +- 预扣、退款、重新预扣和最终结算。 +- 用户、API Key 和单次请求设置优先级。 +- SQLite、MySQL 和 PostgreSQL 的迁移兼容性。 + +### 15.3 端到端与负载测试 + +- 邀请码注册、邮箱验证、登录和 API Key 创建。 +- 开启优化、发起请求和查看实际模型。 +- 管理员配置、模拟、启停规则和查看审计记录。 +- 缓存失效与 Redis 降级。 +- 100 人内测规模下的并发请求和 SSE 长连接。 + +验收基线是：关闭成本优化后，New API 现有接口、协议、渠道、权限和计费行为不退化。 + +## 16. 部署与运维 + +首版 Docker Compose 拓扑： + +```text +Internet + │ + ▼ +Caddy / Nginx + │ + ▼ +New API + CostRouter + ├── PostgreSQL + ├── Redis + └── 上游模型供应商 +``` + +部署要求： + +- 数据库和 Redis 不暴露公网端口。 +- 应用只通过反向代理对外。 +- 自动签发与续期 HTTPS 证书。 +- 每日数据库备份并至少保留七个版本。 +- 镜像使用固定版本，不使用浮动 `latest`。 +- 健康检查覆盖应用、数据库和 Redis。 +- 主机防火墙只开放 80、443 和必要运维入口。 +- 应用保持无状态，未来通过增加实例和负载均衡横向扩容。 + +## 17. 开发阶段 + +### 阶段一：品牌与内测基础 + +- 奶龙工作室主题配置 +- 邀请码注册和邮箱验证 +- 测试额度发放 +- 用户与 API Key 路由偏好 +- Docker Compose 内测环境 + +### 阶段二：成本优化路由核心 + +- `CostRouter` +- 路由规则数据模型 +- 三层模型语义改造 +- 能力过滤与最低成本选择 +- 同模型渠道重试与原模型回退 +- 价格快照和计费一致性 + +### 阶段三：管理与可观测性 + +- 路由规则管理界面 +- 路由模拟器 +- 渠道动态评分和熔断 +- 节省统计 +- 决策审计与告警 + +### 阶段四：兼容性与发布验收 + +- 全渠道回归 +- 三种数据库迁移验证 +- 流式响应与断连测试 +- 并发与异常恢复测试 +- 安全检查 +- 部署和回滚文档 + +## 18. 首版非目标 + +- 在线支付与自动充值 +- 代理商、返佣和分销体系 +- 额外 LLM 提示词分类 +- 机器学习路由模型 +- 多地域部署与 Kubernetes +- 移动端 App +- 无限层级跨模型重试 + +## 19. 验收标准 + +- 受邀用户可以注册、领取测试额度并创建 API Key。 +- 用户可以在 `strict` 与 `optimize_cost` 之间选择。 +- `strict` 模式始终调用用户选择的模型，仅允许同模型渠道切换。 +- `optimize_cost` 模式只选择满足权限、能力、上下文和成本限制的模型。 +- 每次替代都能追溯请求模型、实际模型、规则、渠道、价格快照和结算结果。 +- 替代失败能在首字节前按规定回退，且账务无重复扣减或成本丢失。 +- 用户可以看到实际执行模型与估算节省。 +- 管理员可以配置、模拟、审计路由规则并查看关键指标。 +- PostgreSQL 内测部署稳定，新增迁移同时通过 SQLite、MySQL 和 PostgreSQL 验证。 +- 关闭成本优化时，全渠道兼容性与 New API 基线一致。 diff --git a/docs/superpowers/specs/2026-08-20-multi-instance-admin-intelligent-routing-design.md b/docs/superpowers/specs/2026-08-20-multi-instance-admin-intelligent-routing-design.md new file mode 100644 index 000000000000..b165e63e6a49 --- /dev/null +++ b/docs/superpowers/specs/2026-08-20-multi-instance-admin-intelligent-routing-design.md @@ -0,0 +1,406 @@ +# Multi-Instance Administrator Intelligent Routing Backend Design + +## Status + +- Date: 2026-08-20 +- Scope: administrator control plane, shared runtime state, observability, and operations for intelligent routing +- Runtime: Go, Gin, GORM, Redis +- Database compatibility: SQLite, MySQL 5.7.8+, PostgreSQL 9.6+ + +## Objective + +Turn the existing intelligent-routing execution core into a production-ready administrator subsystem that works consistently across multiple backend instances. + +The subsystem must let an administrator: + +1. create, validate, publish, stage, and roll back immutable routing policies; +2. run shadow or live policies for selected user groups, token groups, and deterministic traffic percentages; +3. inspect shared channel health, model-task quality, route outcomes, real costs, and savings; +4. simulate a request against a draft policy and replay bounded historical samples before publication; +5. isolate or restore a channel and clear shared runtime state; +6. query and acknowledge operational events; +7. retain stable routing behavior when Redis or a database dependency is unavailable. + +The design preserves the current request-facing contract: clients continue to see their requested model, while billing uses the successful execution model and routing details remain administrator-only. + +## Existing System + +The repository already contains: + +- deterministic task classification and candidate planning; +- capability, context, quality, health, and cost filtering; +- bounded multi-attempt execution and response validation; +- in-process health, quality, stickiness, and planning metrics; +- per-request routing detail under `other.admin_info.intelligent_routing`; +- a global `intelligent_routing_setting` configuration object; +- shadow/live execution controlled by global settings. + +The missing production controls are policy history, scoped rollout, multi-instance state, administrator query APIs, manual operations, durable events, actual-cost reporting, and safe dependency degradation. + +## Architecture + +The subsystem is divided into four packages with narrow responsibilities. + +### Policy Control + +Policy Control owns drafts, validation, immutable published versions, rollout targeting, and rollback. It stores durable state in the primary database and publishes invalidation messages through Redis after a successful database commit. + +Each backend instance keeps a read-only in-memory snapshot of the active rollout and referenced policy. Instances refresh the snapshot when they receive an invalidation message and also poll the active revision periodically. Pub/Sub is only an acceleration mechanism; correctness does not depend on receiving every message. + +### Runtime State + +Runtime State owns request-path health windows, quality counters, session stickiness, deterministic rollout assignment, manual isolation, and short-lived metric counters. + +Redis is authoritative for shared transient state when intelligent routing is enabled in multi-instance mode. Updates use atomic Redis commands or bounded Lua scripts. The implementation retains the current in-memory stores behind the same interfaces for tests and explicitly configured single-instance operation. + +### Telemetry + +Telemetry records per-request routing facts in the existing consume-log path and accumulates short-window counters in Redis. Queries combine Redis for current windows with database logs for historical windows. + +Actual savings are calculated only after settlement: + +```text +actual_saving = max(0, requested_model_baseline_charge - execution_model_charge) +``` + +The baseline uses the same token counts and billing conversion rules as settlement. Missing or invalid baseline pricing produces an unavailable saving value rather than a fabricated zero or negative credit. + +### Admin Operations + +Admin Operations exposes stable DTO-based APIs for policy management, simulation, replay, health control, quality inspection, metrics, and operational events. Every mutating endpoint requires root authorization and writes the existing operation audit record. + +## Durable Data Model + +All migrations use GORM and portable scalar columns. JSON documents use `TEXT`; no database-specific JSON operators are required. + +### `intelligent_routing_policies` + +| Field | Purpose | +|---|---| +| `id` | GORM-managed primary key | +| `version` | Unique, monotonically increasing published version; null or zero for drafts | +| `status` | `draft`, `active`, or `archived` | +| `config` | Canonical policy JSON stored as text | +| `checksum` | SHA-256 of canonical policy content | +| `source_version` | Version copied or rolled back from, when applicable | +| `change_note` | Administrator-supplied publication note | +| `created_by` | Administrator user ID | +| `published_by` | Publishing administrator user ID | +| `created_at` | Creation time | +| `updated_at` | Last draft update time | +| `published_at` | Publication time | + +Published rows are immutable. Editing a published policy creates a new draft. Rollback creates and publishes a new version whose content matches the selected historical version; it never mutates history. + +Publication uses a database transaction and `lockForUpdate(tx)` where a row lock is required. It archives the prior active policy, assigns the next version, activates the new policy, updates rollout references when requested, and commits before cache invalidation. + +### `intelligent_routing_rollouts` + +| Field | Purpose | +|---|---| +| `id` | GORM-managed primary key | +| `revision` | Monotonic optimistic-concurrency revision | +| `policy_version` | Published policy used by the rollout | +| `enabled` | Master rollout switch | +| `mode` | `shadow` or `live` | +| `traffic_percent` | Integer from 0 through 100 | +| `user_groups` | Canonical JSON string array | +| `token_groups` | Canonical JSON string array | +| `updated_by` | Administrator user ID | +| `started_at` | Activation time | +| `ended_at` | Disable time | +| `created_at` | Creation time | +| `updated_at` | Modification time | + +Only one current rollout is active. Updates require the caller's last observed revision to prevent one administrator from overwriting another administrator's change. + +### `intelligent_routing_events` + +| Field | Purpose | +|---|---| +| `id` | GORM-managed primary key | +| `event_type` | Stable event identifier | +| `severity` | `info`, `warning`, or `critical` | +| `policy_version` | Associated version, if any | +| `channel_id` | Associated channel, if any | +| `dedupe_key` | Stable key for bounded event coalescing | +| `summary` | Short administrator-facing fallback text | +| `details` | Bounded JSON details stored as text | +| `occurrence_count` | Number of coalesced occurrences | +| `first_seen_at` | First occurrence | +| `last_seen_at` | Latest occurrence | +| `acknowledged_by` | Administrator user ID | +| `acknowledged_at` | Acknowledgement time | +| `resolved_at` | Resolution time | +| `created_at` | Creation time | +| `updated_at` | Modification time | + +Events cover policy publication and rollback, automatic circuit opening and recovery, manual isolation and recovery, repeated no-route or budget exhaustion, Redis degradation, configuration refresh failure, and alert state changes. + +## Policy Document + +The durable policy extends the existing normalized routing configuration. It contains: + +- policy version metadata; +- execution budgets and maximum attempts; +- task-specific quality thresholds; +- model policies and capabilities; +- alert thresholds; +- Redis failure behavior; +- telemetry retention limits. + +Validation rejects: + +- unknown task or capability identifiers; +- duplicate or empty model names; +- invalid tiers, prices, context limits, probabilities, durations, attempt counts, or multipliers; +- a live policy with no eligible candidate model; +- models missing compatible billing configuration; +- rollout percentages outside 0 through 100; +- group names that do not exist at validation time; +- documents exceeding the configured maximum serialized size. + +Validation returns structured errors with field paths and stable codes so the frontend does not parse error strings. + +## Rollout Resolution + +For each supported request: + +1. load the local immutable rollout snapshot; +2. stop if the rollout is disabled; +3. require a match when user-group or token-group allowlists are non-empty; +4. compute a stable bucket from deployment salt, account ID, token ID, and policy version; +5. include buckets lower than `traffic_percent`; +6. plan in shadow or live mode according to the rollout; +7. record the rollout revision and policy version in administrator audit data. + +The stable bucket prevents requests from the same account and token from oscillating between treatment and control. Changing the policy version intentionally reassigns buckets, while rollout-only edits preserve assignments when the version is unchanged. + +## Redis State + +Redis keys are namespaced by deployment identifier and schema version. Raw session identifiers, prompts, token values, and user content are never embedded in keys. + +### Channel health + +Each channel stores a bounded rolling outcome window plus manual isolation metadata. Recording and pruning are atomic. Snapshots return tier, sample count, success count, failure rate, window bounds, isolation state, and last transition time. + +Automatic circuit transitions emit a coalesced durable event asynchronously. Manual isolation always overrides automatic health until an administrator restores the channel. + +### Model-task quality + +Each model-task pair stores bounded successes and samples. Counts have an expiry and are periodically compacted to prevent unbounded keys. Predictions preserve the current cold-start prior and beta smoothing contract. + +### Session stickiness + +Sticky entries store model, channel, task, policy version, expected cost, validation failures, and expiry. Policy-version mismatch invalidates the entry. Two consecutive validation failures remove it atomically. + +### Metrics + +Time-bucketed hashes record planned routes, no-route outcomes, attempts, first-route successes, fallbacks, final failures, validation failures, estimated cost, actual charge, baseline charge, actual savings, and latency aggregates. Cardinality is bounded to approved dimensions: policy version, task, model, channel, outcome, and failure-code family. + +## Dependency Failure Behavior + +Correctness takes priority over routing optimization. + +### Redis unavailable + +- New requests do not execute live intelligent routing in multi-instance mode. +- Requests immediately use the existing channel selector and billing path. +- Shadow planning may run only when it cannot affect execution or billing. +- The instance emits a rate-limited warning and a durable event when the database is available. +- Recovery requires successful Redis health checks and a fresh policy snapshot before live routing resumes. + +The system does not silently switch to per-instance health or stickiness because divergent instance state would make behavior inconsistent. + +### Database unavailable + +- Existing immutable policy snapshots remain usable for a bounded stale interval. +- No publication, rollout mutation, rollback, event acknowledgement, or manual channel operation succeeds. +- After the stale interval, live intelligent routing falls back to the existing selector. + +### Configuration invalidation missed + +Periodic revision polling detects a stale instance. An instance never applies an unvalidated policy document received from Redis. + +## Administrator API + +All routes are placed under `/api/intelligent-routing` and protected by `RootAuth()`. + +### Overview and metrics + +```text +GET /api/intelligent-routing/overview +GET /api/intelligent-routing/metrics +``` + +Overview returns active policy, rollout, dependency health, current alert count, route success summary, actual savings summary, and unhealthy channels. Metrics supports bounded time ranges and filters for policy version, mode, task, model, channel, outcome, and failure family. + +### Policies + +```text +GET /api/intelligent-routing/policies +GET /api/intelligent-routing/policies/:id +POST /api/intelligent-routing/policies +PUT /api/intelligent-routing/policies/:id +POST /api/intelligent-routing/policies/:id/validate +POST /api/intelligent-routing/policies/:id/publish +POST /api/intelligent-routing/policies/:version/rollback +``` + +Draft updates use optimistic concurrency. Publication and rollback require a non-empty change note. + +### Rollout + +```text +GET /api/intelligent-routing/rollout +PUT /api/intelligent-routing/rollout +``` + +The update request includes the last observed revision. Enabling live mode requires a published policy and a successful current validation result. + +### Simulation and replay + +```text +POST /api/intelligent-routing/simulate +POST /api/intelligent-routing/replay +GET /api/intelligent-routing/replay/:job_id +``` + +Simulation accepts a bounded request feature fixture and a draft or published policy identifier. It performs classification and planning without contacting an upstream provider or changing runtime statistics. + +Replay creates a bounded asynchronous job over administrator-visible historical log samples. It returns aggregate eligibility, candidate choice, expected saving, and policy-difference results. It does not reconstruct or expose prompt content. + +### Health and quality + +```text +GET /api/intelligent-routing/channels/health +POST /api/intelligent-routing/channels/:id/isolate +POST /api/intelligent-routing/channels/:id/recover +DELETE /api/intelligent-routing/channels/:id/state +GET /api/intelligent-routing/quality +DELETE /api/intelligent-routing/quality/:model/:task +``` + +Manual operations require a reason and create both operation-audit and routing-event records. + +### Events + +```text +GET /api/intelligent-routing/events +POST /api/intelligent-routing/events/:id/acknowledge +``` + +The list supports pagination, severity, type, acknowledgement state, policy version, channel, and time filters. + +## API Response Rules + +- Responses use explicit DTOs rather than database models or Redis structures. +- Validation errors include a stable `code`, `field`, and localized fallback message key. +- List endpoints enforce maximum page size and maximum time range. +- Model names and event detail strings are length-bounded. +- Redis keys, raw prompts, session fingerprints, tokens, and provider credentials are never returned. +- Mutations are idempotent where practical and reject stale revisions with HTTP 409. + +## Audit + +Existing administrator operation audit gains stable actions: + +- `intelligent_routing.policy.create` +- `intelligent_routing.policy.update` +- `intelligent_routing.policy.publish` +- `intelligent_routing.policy.rollback` +- `intelligent_routing.rollout.update` +- `intelligent_routing.channel.isolate` +- `intelligent_routing.channel.recover` +- `intelligent_routing.channel.reset` +- `intelligent_routing.quality.reset` +- `intelligent_routing.event.acknowledge` + +Audit parameters contain identifiers, versions, revisions, percentages, modes, affected groups, and administrator reasons. They do not contain secrets or request content. + +## Security and Limits + +- Every endpoint requires root authorization. +- All request bodies have explicit size limits. +- Replay concurrency, sample count, and date range are bounded. +- Simulation cannot invoke upstream providers. +- Mutation endpoints use optimistic concurrency and database transactions. +- Operational event details use an allowlisted schema and bounded strings. +- Redis Lua scripts receive typed scalar arguments and do not interpolate user input into script source. +- Metric cardinality excludes user ID, token ID, session ID, and arbitrary error text. + +## Testing + +### Unit tests + +- policy normalization, canonicalization, checksum, and structured validation; +- stable rollout bucketing and group matching; +- Redis key construction and state serialization; +- health transitions, manual isolation precedence, expiry, and recovery; +- quality smoothing and atomic validation-failure invalidation; +- actual savings computation through checked quota helpers; +- DTO filtering, pagination bounds, and failure-code normalization. + +New Go tests use `require` for setup and fatal assertions and `assert` for value checks. + +### Database integration tests + +- draft creation and optimistic update; +- concurrent publication permits only one active version; +- rollback creates a new immutable version; +- rollout revision conflicts return 409; +- event coalescing and acknowledgement; +- migrations and repository behavior on SQLite, MySQL, and PostgreSQL-compatible SQL paths. + +### Redis integration tests + +- two service instances observe shared health, quality, isolation, and stickiness; +- atomic rolling-window updates under concurrent writers; +- expiry and bounded-memory behavior; +- Pub/Sub loss is repaired by revision polling; +- Redis outage disables live intelligent routing and recovery restores it only after refresh. + +### Controller tests + +- root authorization on every route; +- exact success and validation-error DTOs; +- stale revisions, unknown IDs, unavailable dependencies, and bounded query parameters; +- mutation audit records; +- simulation has no upstream or runtime-state side effects. + +### End-to-end verification + +- publish a shadow policy, collect shared metrics, switch a deterministic cohort live, force a channel circuit open, restore it, and roll back; +- verify actual execution-model billing and non-negative savings; +- restart one instance and then all instances without losing durable policy state; +- verify the existing selector handles traffic during Redis failure; +- run the complete root-module test suite and independently build `relaykit` with `GOWORK=off`. + +## Delivery Order + +1. Durable policy, rollout, and event models with migrations and repositories. +2. Policy validation, publication, rollback, and local snapshot refresh. +3. Redis-backed runtime-state interfaces with explicit degradation behavior. +4. Request-path rollout resolution and shared health, quality, and stickiness integration. +5. Settlement-time actual cost and savings telemetry. +6. Administrator policy, rollout, overview, health, quality, and event APIs. +7. Simulation, bounded replay jobs, alert evaluation, and operational documentation. +8. Full multi-instance, database compatibility, regression, build, and rollback verification. + +Each stage remains independently testable and preserves the existing routing behavior until an administrator publishes and enables a rollout. + +## Rollback + +The operational rollback is to disable the active rollout. Existing channel selection resumes without removing policy history or routing audit records. + +The deployment rollback keeps new tables intact, stops new writers, and allows the prior binary to ignore them. Redis keys are versioned and expire naturally; rollback does not require destructive key deletion. + +## Acceptance Criteria + +- Two or more backend instances make rollout, health, quality, and stickiness decisions from shared state. +- Administrators can validate, publish, stage, observe, and roll back policies through stable APIs. +- Live routing stops safely when shared state is unavailable and existing routing continues. +- Published policies and operational events survive process and Redis restarts. +- Actual charges and actual savings are auditable without producing negative credits. +- All mutations are root-authorized, concurrency-safe, and operation-audited. +- SQLite, MySQL, PostgreSQL, the root Go module, and independent `relaykit` builds remain supported. diff --git a/dto/intelligent_routing.go b/dto/intelligent_routing.go new file mode 100644 index 000000000000..275d53ef00d0 --- /dev/null +++ b/dto/intelligent_routing.go @@ -0,0 +1,26 @@ +package dto + +import "time" + +type IntelligentRoutingDraftRequest struct { + Config string `json:"config"` +} + +type IntelligentRoutingDraftUpdateRequest struct { + Config string `json:"config"` + UpdatedAt time.Time `json:"updated_at"` +} + +type IntelligentRoutingPublishRequest struct { + ChangeNote string `json:"change_note"` +} + +type IntelligentRoutingRolloutUpdateRequest struct { + Revision int64 `json:"revision"` + PolicyVersion int `json:"policy_version"` + Enabled bool `json:"enabled"` + Mode string `json:"mode"` + TrafficPercent int `json:"traffic_percent"` + UserGroups []string `json:"user_groups"` + TokenGroups []string `json:"token_groups"` +} diff --git a/main.go b/main.go index 742d15515876..2c600dd0ad06 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ import ( "github.com/QuantumNous/new-api/router" "github.com/QuantumNous/new-api/service" "github.com/QuantumNous/new-api/service/authz" + intelligentrouting "github.com/QuantumNous/new-api/service/intelligent_routing" _ "github.com/QuantumNous/new-api/setting/performance_setting" "github.com/QuantumNous/new-api/setting/ratio_setting" @@ -338,6 +339,12 @@ func InitResources() error { if err != nil { return err } + intelligentrouting.DefaultSharedRuntime.Configure(common.RedisEnabled && common.RDB != nil) + intelligentrouting.DefaultSharedRuntime.SetHealthy(common.RedisEnabled && common.RDB != nil) + if err := intelligentrouting.DefaultPolicyControl.RefreshSnapshot(context.Background()); err != nil { + common.SysError("failed to load intelligent routing policy snapshot: " + err.Error()) + } + intelligentrouting.StartPolicyRefresh(context.Background(), intelligentrouting.DefaultPolicyControl, time.Duration(common.SyncFrequency)*time.Second) perfmetrics.Init() diff --git a/model/channel_cache.go b/model/channel_cache.go index 86c594384d50..6abef00cb02a 100644 --- a/model/channel_cache.go +++ b/model/channel_cache.go @@ -23,6 +23,44 @@ var channelsIDM map[int]*Channel // all channels include dis var channel2advancedCustomConfig map[int]*dto.AdvancedCustomConfig var channelSyncLock sync.RWMutex +// ListEnabledChannelsForRouting returns detached channel values for route planning. +func ListEnabledChannelsForRouting(group, requestPath string) []*Channel { + channelSyncLock.RLock() + defer channelSyncLock.RUnlock() + modelChannels := group2model2channels[group] + channelModels := collectRoutingChannelModels(modelChannels, requestPath, filterChannelsByRequestPathAndModel) + channelIDs := make([]int, 0, len(channelModels)) + for channelID := range channelModels { + channelIDs = append(channelIDs, channelID) + } + sort.Ints(channelIDs) + result := make([]*Channel, 0, len(channelIDs)) + for _, channelID := range channelIDs { + channel, ok := channelsIDM[channelID] + if !ok || channel == nil || channel.Status != common.ChannelStatusEnabled { + continue + } + copy := *channel + copy.Keys = append([]string(nil), channel.Keys...) + copy.Models = strings.Join(channelModels[channelID], ",") + result = append(result, ©) + } + return result +} + +func collectRoutingChannelModels(modelChannels map[string][]int, requestPath string, filter func([]int, string, string) []int) map[int][]string { + result := make(map[int][]string) + for modelName, channelIDs := range modelChannels { + for _, channelID := range filter(channelIDs, requestPath, modelName) { + result[channelID] = append(result[channelID], modelName) + } + } + for channelID := range result { + sort.Strings(result[channelID]) + } + return result +} + func InitChannelCache() { if !common.MemoryCacheEnabled { InvalidatePricingCache() diff --git a/model/channel_cache_routing_test.go b/model/channel_cache_routing_test.go new file mode 100644 index 000000000000..123a4e785f05 --- /dev/null +++ b/model/channel_cache_routing_test.go @@ -0,0 +1,19 @@ +package model + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCollectRoutingChannelModelsKeepsOnlyPathCompatibleModels(t *testing.T) { + models := map[string][]int{"chat-only": {7}, "responses-only": {7}, "other": {8}} + got := collectRoutingChannelModels(models, "/v1/responses", func(ids []int, path, model string) []int { + if model == "chat-only" { + return nil + } + return ids + }) + assert.Equal(t, []string{"responses-only"}, got[7]) + assert.Equal(t, []string{"other"}, got[8]) +} diff --git a/model/intelligent_routing_policy.go b/model/intelligent_routing_policy.go new file mode 100644 index 000000000000..62b1434bf197 --- /dev/null +++ b/model/intelligent_routing_policy.go @@ -0,0 +1,252 @@ +package model + +import ( + "errors" + "time" + + "gorm.io/gorm" +) + +const ( + IntelligentRoutingPolicyDraft = "draft" + IntelligentRoutingPolicyActive = "active" + IntelligentRoutingPolicyArchived = "archived" + + IntelligentRoutingModeShadow = "shadow" + IntelligentRoutingModeLive = "live" +) + +var ( + ErrIntelligentRoutingPolicyNotFound = errors.New("intelligent routing policy not found") + ErrIntelligentRoutingRolloutNotFound = errors.New("intelligent routing rollout not found") + ErrIntelligentRoutingPolicyImmutable = errors.New("published intelligent routing policy is immutable") + ErrIntelligentRoutingRevisionConflict = errors.New("intelligent routing revision conflict") +) + +type IntelligentRoutingPolicy struct { + Id int64 `json:"id" gorm:"primaryKey"` + Version int `json:"version" gorm:"index"` + Status string `json:"status" gorm:"type:varchar(16);index"` + Config string `json:"config" gorm:"type:text"` + Checksum string `json:"checksum" gorm:"type:varchar(64)"` + SourceVersion int `json:"source_version"` + ChangeNote string `json:"change_note" gorm:"type:varchar(500)"` + CreatedBy int `json:"created_by"` + PublishedBy int `json:"published_by"` + PublishedAt *time.Time `json:"published_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type IntelligentRoutingRollout struct { + Id int64 `json:"id" gorm:"primaryKey"` + Revision int64 `json:"revision"` + PolicyVersion int `json:"policy_version"` + Enabled bool `json:"enabled"` + Mode string `json:"mode" gorm:"type:varchar(16)"` + TrafficPercent int `json:"traffic_percent"` + UserGroups string `json:"user_groups" gorm:"type:text"` + TokenGroups string `json:"token_groups" gorm:"type:text"` + UpdatedBy int `json:"updated_by"` + StartedAt *time.Time `json:"started_at"` + EndedAt *time.Time `json:"ended_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func CreateIntelligentRoutingDraft(policy IntelligentRoutingPolicy) (IntelligentRoutingPolicy, error) { + policy.Id = 0 + policy.Version = 0 + policy.Status = IntelligentRoutingPolicyDraft + policy.PublishedBy = 0 + policy.PublishedAt = nil + err := DB.Create(&policy).Error + return policy, err +} + +func UpdateIntelligentRoutingDraft(id int64, updatedAt time.Time, config, checksum string) (IntelligentRoutingPolicy, error) { + var policy IntelligentRoutingPolicy + err := DB.Transaction(func(tx *gorm.DB) error { + if err := lockForUpdate(tx).Where("id = ?", id).First(&policy).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return ErrIntelligentRoutingPolicyNotFound + } + return err + } + if policy.Status != IntelligentRoutingPolicyDraft { + return ErrIntelligentRoutingPolicyImmutable + } + result := tx.Model(&IntelligentRoutingPolicy{}). + Where("id = ? AND status = ? AND updated_at = ?", id, IntelligentRoutingPolicyDraft, updatedAt). + Updates(map[string]any{"config": config, "checksum": checksum}) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return ErrIntelligentRoutingRevisionConflict + } + return tx.Where("id = ?", id).First(&policy).Error + }) + return policy, err +} + +func ListIntelligentRoutingPolicies(offset, limit int) ([]IntelligentRoutingPolicy, int64, error) { + var policies []IntelligentRoutingPolicy + var total int64 + if err := DB.Model(&IntelligentRoutingPolicy{}).Count(&total).Error; err != nil { + return nil, 0, err + } + err := DB.Order("id DESC").Offset(offset).Limit(limit).Find(&policies).Error + return policies, total, err +} + +func GetIntelligentRoutingPolicy(id int64) (IntelligentRoutingPolicy, error) { + var policy IntelligentRoutingPolicy + err := DB.Where("id = ?", id).First(&policy).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + err = ErrIntelligentRoutingPolicyNotFound + } + return policy, err +} + +func GetActiveIntelligentRoutingPolicy() (IntelligentRoutingPolicy, error) { + var policy IntelligentRoutingPolicy + err := DB.Where("status = ?", IntelligentRoutingPolicyActive).Order("version DESC").First(&policy).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + err = ErrIntelligentRoutingPolicyNotFound + } + return policy, err +} + +func GetIntelligentRoutingPolicyByVersion(version int) (IntelligentRoutingPolicy, error) { + var policy IntelligentRoutingPolicy + err := DB.Where("version = ?", version).First(&policy).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + err = ErrIntelligentRoutingPolicyNotFound + } + return policy, err +} + +func PublishIntelligentRoutingPolicy(id int64, administratorID int, changeNote string) (IntelligentRoutingPolicy, error) { + var published IntelligentRoutingPolicy + err := DB.Transaction(func(tx *gorm.DB) error { + var draft IntelligentRoutingPolicy + if err := lockForUpdate(tx).Where("id = ?", id).First(&draft).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return ErrIntelligentRoutingPolicyNotFound + } + return err + } + if draft.Status != IntelligentRoutingPolicyDraft { + return ErrIntelligentRoutingPolicyImmutable + } + + var latest IntelligentRoutingPolicy + latestErr := lockForUpdate(tx).Order("version DESC").First(&latest).Error + if latestErr != nil && !errors.Is(latestErr, gorm.ErrRecordNotFound) { + return latestErr + } + nextVersion := latest.Version + 1 + if err := tx.Model(&IntelligentRoutingPolicy{}). + Where("status = ?", IntelligentRoutingPolicyActive). + Update("status", IntelligentRoutingPolicyArchived).Error; err != nil { + return err + } + now := time.Now() + updates := map[string]any{ + "version": nextVersion, "status": IntelligentRoutingPolicyActive, "change_note": changeNote, + "published_by": administratorID, "published_at": &now, + } + result := tx.Model(&IntelligentRoutingPolicy{}). + Where("id = ? AND status = ?", id, IntelligentRoutingPolicyDraft). + Updates(updates) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return ErrIntelligentRoutingRevisionConflict + } + return tx.Where("id = ?", id).First(&published).Error + }) + return published, err +} + +func RollbackIntelligentRoutingPolicy(sourceVersion int, administratorID int, changeNote string) (IntelligentRoutingPolicy, error) { + var rolledBack IntelligentRoutingPolicy + err := DB.Transaction(func(tx *gorm.DB) error { + var source IntelligentRoutingPolicy + if err := lockForUpdate(tx).Where("version = ?", sourceVersion).First(&source).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return ErrIntelligentRoutingPolicyNotFound + } + return err + } + var latest IntelligentRoutingPolicy + latestErr := lockForUpdate(tx).Order("version DESC").First(&latest).Error + if latestErr != nil && !errors.Is(latestErr, gorm.ErrRecordNotFound) { + return latestErr + } + if err := tx.Model(&IntelligentRoutingPolicy{}). + Where("status = ?", IntelligentRoutingPolicyActive). + Update("status", IntelligentRoutingPolicyArchived).Error; err != nil { + return err + } + now := time.Now() + rolledBack = IntelligentRoutingPolicy{ + Version: latest.Version + 1, Status: IntelligentRoutingPolicyActive, Config: source.Config, + Checksum: source.Checksum, SourceVersion: sourceVersion, ChangeNote: changeNote, + CreatedBy: administratorID, PublishedBy: administratorID, PublishedAt: &now, + } + return tx.Create(&rolledBack).Error + }) + return rolledBack, err +} + +func GetIntelligentRoutingRollout() (IntelligentRoutingRollout, error) { + var rollout IntelligentRoutingRollout + err := DB.First(&rollout, 1).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + err = ErrIntelligentRoutingRolloutNotFound + } + return rollout, err +} + +func UpdateIntelligentRoutingRollout(expectedRevision int64, next IntelligentRoutingRollout) (IntelligentRoutingRollout, error) { + var stored IntelligentRoutingRollout + err := DB.Transaction(func(tx *gorm.DB) error { + var current IntelligentRoutingRollout + err := lockForUpdate(tx).First(¤t, 1).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + if expectedRevision != 0 { + return ErrIntelligentRoutingRevisionConflict + } + next.Id = 1 + next.Revision = 1 + if err := tx.Create(&next).Error; err != nil { + return err + } + stored = next + return nil + } + if err != nil { + return err + } + if current.Revision != expectedRevision { + return ErrIntelligentRoutingRevisionConflict + } + updates := map[string]any{ + "revision": expectedRevision + 1, "policy_version": next.PolicyVersion, "enabled": next.Enabled, + "mode": next.Mode, "traffic_percent": next.TrafficPercent, "user_groups": next.UserGroups, + "token_groups": next.TokenGroups, "updated_by": next.UpdatedBy, "started_at": next.StartedAt, "ended_at": next.EndedAt, + } + result := tx.Model(&IntelligentRoutingRollout{}).Where("id = ? AND revision = ?", current.Id, expectedRevision).Updates(updates) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return ErrIntelligentRoutingRevisionConflict + } + return tx.First(&stored, current.Id).Error + }) + return stored, err +} diff --git a/model/intelligent_routing_policy_test.go b/model/intelligent_routing_policy_test.go new file mode 100644 index 000000000000..1821fd429b2c --- /dev/null +++ b/model/intelligent_routing_policy_test.go @@ -0,0 +1,94 @@ +package model + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func resetIntelligentRoutingPolicyTables(t *testing.T) { + t.Helper() + require.NoError(t, DB.AutoMigrate(&IntelligentRoutingPolicy{}, &IntelligentRoutingRollout{})) + require.NoError(t, DB.Exec("DELETE FROM intelligent_routing_rollouts").Error) + require.NoError(t, DB.Exec("DELETE FROM intelligent_routing_policies").Error) + t.Cleanup(func() { + require.NoError(t, DB.Exec("DELETE FROM intelligent_routing_rollouts").Error) + require.NoError(t, DB.Exec("DELETE FROM intelligent_routing_policies").Error) + }) +} + +func TestIntelligentRoutingPolicyDraftLifecycle(t *testing.T) { + resetIntelligentRoutingPolicyTables(t) + + draft, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{ + Status: IntelligentRoutingPolicyDraft, Config: `{"enabled":false}`, Checksum: "sum", CreatedBy: 11, + }) + require.NoError(t, err) + assert.Equal(t, IntelligentRoutingPolicyDraft, draft.Status) + + stored, err := GetIntelligentRoutingPolicy(draft.Id) + require.NoError(t, err) + assert.Equal(t, draft.Id, stored.Id) + + stored.Status = IntelligentRoutingPolicyActive + require.NoError(t, DB.Save(&stored).Error) + _, err = UpdateIntelligentRoutingDraft(stored.Id, stored.UpdatedAt, `{"enabled":true}`, "next") + assert.ErrorIs(t, err, ErrIntelligentRoutingPolicyImmutable) +} + +func TestIntelligentRoutingPolicyPublishArchivesPriorVersion(t *testing.T) { + resetIntelligentRoutingPolicyTables(t) + + first, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{Config: `{"enabled":false}`, Checksum: "one", CreatedBy: 1}) + require.NoError(t, err) + first, err = PublishIntelligentRoutingPolicy(first.Id, 1, "first") + require.NoError(t, err) + assert.Equal(t, 1, first.Version) + assert.Equal(t, IntelligentRoutingPolicyActive, first.Status) + + second, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{Config: `{"enabled":true}`, Checksum: "two", CreatedBy: 2}) + require.NoError(t, err) + second, err = PublishIntelligentRoutingPolicy(second.Id, 2, "second") + require.NoError(t, err) + assert.Equal(t, 2, second.Version) + + first, err = GetIntelligentRoutingPolicy(first.Id) + require.NoError(t, err) + assert.Equal(t, IntelligentRoutingPolicyArchived, first.Status) +} + +func TestIntelligentRoutingPolicyRollbackCreatesNewVersion(t *testing.T) { + resetIntelligentRoutingPolicyTables(t) + + first, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{Config: `{"enabled":false}`, Checksum: "one", CreatedBy: 1}) + require.NoError(t, err) + _, err = PublishIntelligentRoutingPolicy(first.Id, 1, "first") + require.NoError(t, err) + second, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{Config: `{"enabled":true}`, Checksum: "two", CreatedBy: 2}) + require.NoError(t, err) + _, err = PublishIntelligentRoutingPolicy(second.Id, 2, "second") + require.NoError(t, err) + + rolledBack, err := RollbackIntelligentRoutingPolicy(1, 3, "restore first") + require.NoError(t, err) + assert.Equal(t, 3, rolledBack.Version) + assert.Equal(t, 1, rolledBack.SourceVersion) + assert.Equal(t, `{"enabled":false}`, rolledBack.Config) +} + +func TestIntelligentRoutingRolloutRejectsStaleRevision(t *testing.T) { + resetIntelligentRoutingPolicyTables(t) + + rollout, err := UpdateIntelligentRoutingRollout(0, IntelligentRoutingRollout{PolicyVersion: 1, Enabled: true, Mode: IntelligentRoutingModeShadow, TrafficPercent: 25}) + require.NoError(t, err) + assert.Equal(t, int64(1), rollout.Revision) + + _, err = UpdateIntelligentRoutingRollout(0, IntelligentRoutingRollout{PolicyVersion: 1, Enabled: false, Mode: IntelligentRoutingModeShadow}) + assert.True(t, errors.Is(err, ErrIntelligentRoutingRevisionConflict)) + + stored, err := GetIntelligentRoutingRollout() + require.NoError(t, err) + assert.True(t, stored.Enabled) +} diff --git a/model/main.go b/model/main.go index 21445593e54e..4dfda76f08d0 100644 --- a/model/main.go +++ b/model/main.go @@ -290,6 +290,8 @@ func migrateDB() error { &SystemInstance{}, &SystemTask{}, &SystemTaskLock{}, + &IntelligentRoutingPolicy{}, + &IntelligentRoutingRollout{}, &CasbinRule{}, &AuthzRole{}, ) @@ -353,6 +355,8 @@ func migrateDBFast() error { {&SystemInstance{}, "SystemInstance"}, {&SystemTask{}, "SystemTask"}, {&SystemTaskLock{}, "SystemTaskLock"}, + {&IntelligentRoutingPolicy{}, "IntelligentRoutingPolicy"}, + {&IntelligentRoutingRollout{}, "IntelligentRoutingRollout"}, } // 动态计算migration数量，确保errChan缓冲区足够大 errChan := make(chan error, len(migrations)) diff --git a/model/option.go b/model/option.go index e7fda5231be7..9c6b8f7d3e6b 100644 --- a/model/option.go +++ b/model/option.go @@ -8,6 +8,7 @@ import ( "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/setting" "github.com/QuantumNous/new-api/setting/config" + "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" "github.com/QuantumNous/new-api/setting/operation_setting" "github.com/QuantumNous/new-api/setting/performance_setting" "github.com/QuantumNous/new-api/setting/ratio_setting" @@ -633,6 +634,10 @@ func handleConfigUpdate(key, value string) bool { // 特定配置的后处理 if configName == "performance_setting" { performance_setting.UpdateAndSync() + } else if configName == "intelligent_routing_setting" { + if err := intelligent_routing_setting.UpdateAndSync(); err != nil { + common.SysError("failed to normalize intelligent routing setting: " + err.Error()) + } } else if configName == "billing_setting" { InvalidatePricingCache() ratio_setting.InvalidateExposedDataCache() diff --git a/relay/channel/api_request_getbody_test.go b/relay/channel/api_request_getbody_test.go index 9a2de73337cd..a984ffb1d897 100644 --- a/relay/channel/api_request_getbody_test.go +++ b/relay/channel/api_request_getbody_test.go @@ -334,6 +334,21 @@ func writeH2TestResponse(framer *http2.Framer, streamID uint32) error { return framer.WriteData(streamID, true, []byte(`{}`)) } +func writeH2TestPingBarrier(framer *http2.Framer, data [8]byte) error { + if err := framer.WritePing(false, data); err != nil { + return err + } + for { + frame, err := framer.ReadFrame() + if err != nil { + return err + } + if ack, ok := frame.(*http2.PingFrame); ok && ack.Flags.Has(http2.FlagPingAck) && ack.Data == data { + return nil + } + } +} + func awaitH2ServerResult(t *testing.T, resultCh <-chan h2ServerResult) h2ServerResult { t.Helper() select { @@ -380,6 +395,10 @@ func runResetOnFirstStreamServer(ln net.Listener, expectRetry bool) <-chan h2Ser return } if !expectRetry { + if err := writeH2TestPingBarrier(framer, [8]byte{'r', 'e', 's', 'e', 't'}); err != nil { + res.err = err + return + } break attempts } continue @@ -400,6 +419,12 @@ func runGoAwayAfterFirstRequestServer(ln net.Listener) <-chan h2ServerResult { res := h2ServerResult{} defer func() { resCh <- res }() + var drainingConn net.Conn + defer func() { + if drainingConn != nil { + drainingConn.Close() + } + }() for attempt := 0; attempt < 2; attempt++ { conn, framer, err := acceptH2TestConnection(ln) if err != nil { @@ -417,15 +442,26 @@ func runGoAwayAfterFirstRequestServer(ln net.Listener) <-chan h2ServerResult { if attempt == 0 { err = framer.WriteGoAway(0, http2.ErrCodeNo, nil) - conn.Close() if err != nil { + conn.Close() res.err = err return } + // Keep the first connection alive until the transport opens the + // replacement connection. That proves GOAWAY was consumed and avoids + // racing the frame with an immediate Windows TCP reset. + drainingConn = conn continue } + if drainingConn != nil { + drainingConn.Close() + drainingConn = nil + } err = writeH2TestResponse(framer, streamID) + if err == nil { + err = writeH2TestPingBarrier(framer, [8]byte{'r', 'e', 's', 'p'}) + } conn.Close() if err != nil { res.err = err diff --git a/relay/channel/openai/model_identity_test.go b/relay/channel/openai/model_identity_test.go new file mode 100644 index 000000000000..2cf4bb3c6c65 --- /dev/null +++ b/relay/channel/openai/model_identity_test.go @@ -0,0 +1,47 @@ +package openai + +import ( + "testing" + + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/types" + hosttypes "github.com/QuantumNous/new-api/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeOpenAIResponseModelUsesRequestedIdentity(t *testing.T) { + info := &relaycommon.RelayInfo{OriginModelName: "requested", ExecutionModelName: "cheap", RelayFormat: types.RelayFormatOpenAI} + got, err := normalizeOpenAIResponseModel([]byte(`{"id":"x","model":"cheap","choices":[]}`), info) + require.NoError(t, err) + assert.JSONEq(t, `{"id":"x","model":"requested","choices":[]}`, string(got)) +} + +func TestValidateIntelligentRoutingResponseRunsOnlyForLivePlan(t *testing.T) { + request := &dto.GeneralOpenAIRequest{} + live := &relaycommon.RelayInfo{Request: request, RelayFormat: types.RelayFormatOpenAI, IntelligentRoutePlan: &hosttypes.IntelligentRoutePlan{}} + assert.Error(t, validateIntelligentRoutingResponse(live, []byte(`{"choices":[]}`))) + assert.NoError(t, validateIntelligentRoutingResponse(&relaycommon.RelayInfo{Request: request, RelayFormat: types.RelayFormatOpenAI}, []byte(`{"choices":[]}`))) +} + +func TestNormalizeOpenAIResponseModelLeavesOrdinaryResponseUntouched(t *testing.T) { + body := []byte(`{"model":"requested"}`) + got, err := normalizeOpenAIResponseModel(body, &relaycommon.RelayInfo{OriginModelName: "requested"}) + require.NoError(t, err) + assert.Equal(t, body, got) +} + +func TestNormalizeResponsesAPIModelUsesRequestedIdentity(t *testing.T) { + info := &relaycommon.RelayInfo{OriginModelName: "requested", ExecutionModelName: "cheap", RelayFormat: types.RelayFormatOpenAIResponses} + got, err := normalizeOpenAIResponseModel([]byte(`{"model":"cheap","output":[]}`), info) + require.NoError(t, err) + assert.JSONEq(t, `{"model":"requested","output":[]}`, string(got)) +} + +func TestNormalizeResponsesCompactionModelUsesRequestedIdentity(t *testing.T) { + info := &relaycommon.RelayInfo{OriginModelName: "requested", ExecutionModelName: "cheap", RelayFormat: types.RelayFormatOpenAIResponsesCompaction} + got, err := normalizeOpenAIResponseModel([]byte(`{"model":"cheap","output":[]}`), info) + require.NoError(t, err) + assert.JSONEq(t, `{"model":"requested","output":[]}`, string(got)) +} diff --git a/relay/channel/openai/relay-openai.go b/relay/channel/openai/relay-openai.go index 9a0619eb27f5..2e4f877be2f2 100644 --- a/relay/channel/openai/relay-openai.go +++ b/relay/channel/openai/relay-openai.go @@ -16,6 +16,7 @@ import ( "github.com/QuantumNous/new-api/relaykit/relayconvert" "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/service" + intelligentrouting "github.com/QuantumNous/new-api/service/intelligent_routing" "github.com/gin-gonic/gin" ) @@ -24,6 +25,17 @@ func sendStreamData(c *gin.Context, info *relaycommon.RelayInfo, data string, fo if data == "" { return nil } + if info != nil && info.ExecutionModelName != "" && info.ExecutionModelName != info.OriginModelName && info.RelayFormat == types.RelayFormatOpenAI { + var response dto.ChatCompletionsStreamResponse + if err := common.UnmarshalJsonStr(data, &response); err == nil { + response.Model = info.OriginModelName + normalized, err := common.Marshal(response) + if err != nil { + return err + } + data = string(normalized) + } + } if !forceFormat && !thinkToContent { return helper.StringData(c, data) @@ -330,8 +342,35 @@ func OpenaiHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respo } responseBody = geminiRespStr } + if err = validateIntelligentRoutingResponse(info, responseBody); err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + responseBody, err = normalizeOpenAIResponseModel(responseBody, info) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + } service.IOCopyBytesGracefully(c, resp, responseBody) return &simpleResponse.Usage, nil } + +func normalizeOpenAIResponseModel(body []byte, info *relaycommon.RelayInfo) ([]byte, error) { + if info == nil || (info.RelayFormat != types.RelayFormatOpenAI && info.RelayFormat != types.RelayFormatOpenAIResponses && info.RelayFormat != types.RelayFormatOpenAIResponsesCompaction) || info.ExecutionModelName == "" || + info.ExecutionModelName == info.OriginModelName || info.OriginModelName == "" { + return body, nil + } + var response map[string]interface{} + if err := common.Unmarshal(body, &response); err != nil { + return nil, err + } + response["model"] = info.OriginModelName + return common.Marshal(response) +} + +func validateIntelligentRoutingResponse(info *relaycommon.RelayInfo, body []byte) error { + if info == nil || info.IntelligentRoutePlan == nil || info.IntelligentRouteShadow { + return nil + } + return intelligentrouting.ValidateResponse(info.Request, info.RelayFormat, body) +} diff --git a/relay/channel/openai/relay_responses.go b/relay/channel/openai/relay_responses.go index ceca1af3b381..1240a250ddd8 100644 --- a/relay/channel/openai/relay_responses.go +++ b/relay/channel/openai/relay_responses.go @@ -33,6 +33,13 @@ func OaiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http if oaiError := responsesResponse.GetOpenAIError(); oaiError != nil && oaiError.Type != "" { return nil, types.WithOpenAIError(*oaiError, resp.StatusCode) } + if err = validateIntelligentRoutingResponse(info, responseBody); err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + responseBody, err = normalizeOpenAIResponseModel(responseBody, info) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + } // 写入新的 response body service.IOCopyBytesGracefully(c, resp, responseBody) @@ -94,6 +101,15 @@ func OaiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp sr.Error(err) return } + if info.ExecutionModelName != "" && info.ExecutionModelName != info.OriginModelName && streamResponse.Response != nil { + streamResponse.Response.Model = info.OriginModelName + normalized, err := common.Marshal(streamResponse) + if err != nil { + sr.Error(err) + return + } + data = string(normalized) + } sendResponsesStreamData(c, streamResponse, data) switch streamResponse.Type { case "response.completed", "response.done": diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index b0bb19bdca3b..215a45d1ea9a 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -98,6 +98,7 @@ type RelayInfo struct { UsePrice bool RelayMode int OriginModelName string + ExecutionModelName string RequestURLPath string RequestHeaders map[string]string ShouldIncludeUsage bool @@ -149,7 +150,19 @@ type RelayInfo struct { UseRuntimeHeadersOverride bool ParamOverrideAudit []string - PriceData hosttypes.PriceData + PriceData hosttypes.PriceData + IntelligentRoutePlan *hosttypes.IntelligentRoutePlan + IntelligentRouteError string + IntelligentRouteShadow bool + IntelligentRouteLive bool + IntelligentRoutePolicyVersion int + IntelligentRouteRolloutRevision int64 + IntelligentRouteRolloutBucket int + IntelligentRouteRolloutMode string + IntelligentRouteAttempt int + IntelligentRouteSessionKey string + IntelligentRouteTask string + IntelligentRouteAttempts []hosttypes.IntelligentRouteAttempt // QuotaClamp is set (non-nil) when a quota conversion saturated at the // int32 bound (or NaN fallback) while computing this request's charge. @@ -739,6 +752,33 @@ func (info *RelayInfo) GetOriginModelName() string { return info.OriginModelName } +func (info *RelayInfo) SetExecutionModelName(modelName string) { + if info == nil { + return + } + info.ExecutionModelName = modelName + if info.ChannelMeta != nil { + info.ChannelMeta.UpstreamModelName = modelName + } + if info.Request != nil { + info.Request.SetModelName(modelName) + } +} + +func (info *RelayInfo) GetExecutionModelName() string { + if info == nil { + return "" + } + if info.ExecutionModelName != "" { + return info.ExecutionModelName + } + return info.OriginModelName +} + +func (info *RelayInfo) GetBillingModelName() string { + return info.GetExecutionModelName() +} + func (info *RelayInfo) GetUpstreamModelName() string { if info == nil || info.ChannelMeta == nil { return "" diff --git a/relay/common/relay_info_test.go b/relay/common/relay_info_test.go index 42a0f8567bfe..2892e975266f 100644 --- a/relay/common/relay_info_test.go +++ b/relay/common/relay_info_test.go @@ -45,6 +45,21 @@ func TestRelayInfoGetFinalRequestRelayFormatNilReceiver(t *testing.T) { require.Equal(t, types.RelayFormat(""), info.GetFinalRequestRelayFormat()) } +func TestRelayInfoExecutionModelPreservesRequestedIdentity(t *testing.T) { + request := &dto.GeneralOpenAIRequest{Model: "requested-model"} + info := &RelayInfo{OriginModelName: "requested-model", Request: request, ChannelMeta: &ChannelMeta{}} + info.SetExecutionModelName("cheap-model") + assert.Equal(t, "requested-model", info.OriginModelName) + assert.Equal(t, "cheap-model", info.GetExecutionModelName()) + assert.Equal(t, "cheap-model", info.GetBillingModelName()) + assert.Equal(t, "cheap-model", request.Model) +} + +func TestRelayInfoBillingModelFallsBackToRequestedModel(t *testing.T) { + info := &RelayInfo{OriginModelName: "requested-model"} + assert.Equal(t, "requested-model", info.GetBillingModelName()) +} + func TestRelayInfoMetaTypedNilReceiver(t *testing.T) { var info *RelayInfo var meta convmeta.Meta = info diff --git a/relay/helper/price.go b/relay/helper/price.go index b9ae819bf57f..b5363f8d6846 100644 --- a/relay/helper/price.go +++ b/relay/helper/price.go @@ -71,12 +71,13 @@ func HandleGroupRatio(ctx *gin.Context, relayInfo *relaycommon.RelayInfo) hostty } func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens int, meta *types.TokenCountMeta) (hosttypes.PriceData, error) { - modelPrice, usePrice := ratio_setting.GetModelPrice(info.OriginModelName, false) + modelName := info.GetBillingModelName() + modelPrice, usePrice := ratio_setting.GetModelPrice(modelName, false) groupRatioInfo := HandleGroupRatio(c, info) // Check if this model uses tiered_expr billing - if billing_setting.GetBillingMode(info.OriginModelName) == billing_setting.BillingModeTieredExpr { + if billing_setting.GetBillingMode(modelName) == billing_setting.BillingModeTieredExpr { return modelPriceHelperTiered(c, info, promptTokens, meta, groupRatioInfo) } @@ -98,7 +99,7 @@ func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens } var success bool var matchName string - modelRatio, success, matchName = ratio_setting.GetModelRatio(info.OriginModelName) + modelRatio, success, matchName = ratio_setting.GetModelRatio(modelName) if !success { acceptUnsetRatio := false if info.UserSetting.AcceptUnsetRatioModel { @@ -108,15 +109,15 @@ func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens return hosttypes.PriceData{}, modelPriceNotConfiguredError(matchName, info.UserId) } } - completionRatio = ratio_setting.GetCompletionRatio(info.OriginModelName) - cacheRatio, _ = ratio_setting.GetCacheRatio(info.OriginModelName) - cacheCreationRatio, _ = ratio_setting.GetCreateCacheRatio(info.OriginModelName) + completionRatio = ratio_setting.GetCompletionRatio(modelName) + cacheRatio, _ = ratio_setting.GetCacheRatio(modelName) + cacheCreationRatio, _ = ratio_setting.GetCreateCacheRatio(modelName) cacheCreationRatio5m = cacheCreationRatio // 固定1h和5min缓存写入价格的比例 cacheCreationRatio1h = cacheCreationRatio * claudeCacheCreation1hMultiplier - imageRatio, _ = ratio_setting.GetImageRatio(info.OriginModelName) - audioRatio = ratio_setting.GetAudioRatio(info.OriginModelName) - audioCompletionRatio = ratio_setting.GetAudioCompletionRatio(info.OriginModelName) + imageRatio, _ = ratio_setting.GetImageRatio(modelName) + audioRatio = ratio_setting.GetAudioRatio(modelName) + audioCompletionRatio = ratio_setting.GetAudioCompletionRatio(modelName) ratio := modelRatio * groupRatioInfo.GroupRatio quota, err := common.QuotaFromFloatStrict(float64(preConsumedTokens) * ratio) if err != nil { @@ -267,9 +268,10 @@ func HasModelBillingConfig(modelName string) bool { } func modelPriceHelperTiered(c *gin.Context, info *relaycommon.RelayInfo, promptTokens int, meta *types.TokenCountMeta, groupRatioInfo hosttypes.GroupRatioInfo) (hosttypes.PriceData, error) { - exprStr, ok := billing_setting.GetBillingExpr(info.OriginModelName) + modelName := info.GetBillingModelName() + exprStr, ok := billing_setting.GetBillingExpr(modelName) if !ok { - return hosttypes.PriceData{}, fmt.Errorf("model %s is configured as tiered_expr but has no billing expression", info.OriginModelName) + return hosttypes.PriceData{}, fmt.Errorf("model %s is configured as tiered_expr but has no billing expression", modelName) } estimatedCompletionTokens := meta.MaxTokens @@ -288,7 +290,7 @@ func modelPriceHelperTiered(c *gin.Context, info *relaycommon.RelayInfo, promptT Len: float64(promptTokens), }, requestInput) if err != nil { - return hosttypes.PriceData{}, fmt.Errorf("model %s tiered expr run failed: %w", info.OriginModelName, err) + return hosttypes.PriceData{}, fmt.Errorf("model %s tiered expr run failed: %w", modelName, err) } // Expression coefficients are $/1M tokens prices; convert to quota the same way per-call billing does. @@ -309,7 +311,7 @@ func modelPriceHelperTiered(c *gin.Context, info *relaycommon.RelayInfo, promptT exprHash := billingexpr.ExprHashString(exprStr) snapshot := &billingexpr.BillingSnapshot{ BillingMode: billing_setting.BillingModeTieredExpr, - ModelName: info.OriginModelName, + ModelName: modelName, ExprString: exprStr, ExprHash: exprHash, GroupRatio: groupRatioInfo.GroupRatio, @@ -330,7 +332,7 @@ func modelPriceHelperTiered(c *gin.Context, info *relaycommon.RelayInfo, promptT QuotaToPreConsume: preConsumedQuota, } - logger.LogDebug(c, "model_price_helper_tiered result: model=%s preConsume=%d quotaBeforeGroup=%.2f groupRatio=%.2f tier=%s", info.OriginModelName, preConsumedQuota, quotaBeforeGroup, groupRatioInfo.GroupRatio, trace.MatchedTier) + logger.LogDebug(c, "model_price_helper_tiered result: model=%s preConsume=%d quotaBeforeGroup=%.2f groupRatio=%.2f tier=%s", modelName, preConsumedQuota, quotaBeforeGroup, groupRatioInfo.GroupRatio, trace.MatchedTier) info.PriceData = priceData return priceData, nil diff --git a/relay/helper/price_test.go b/relay/helper/price_test.go index 0f28b5a424c5..64419f067bef 100644 --- a/relay/helper/price_test.go +++ b/relay/helper/price_test.go @@ -272,3 +272,17 @@ func TestModelPriceHelperRequestBillingRatiosOnlyApplyToFixedPrice(t *testing.T) require.Equal(t, common.QuotaClampOverflow, clamp.Kind) require.Nil(t, info.Billing) } + +func TestModelPriceHelperUsesExecutionModelForBilling(t *testing.T) { + saved := ratio_setting.ModelPrice2JSONString() + t.Cleanup(func() { require.NoError(t, ratio_setting.UpdateModelPriceByJSONString(saved)) }) + prices, err := common.Marshal(map[string]float64{"requested-model": 1, "cheap-model": 0.25}) + require.NoError(t, err) + require.NoError(t, ratio_setting.UpdateModelPriceByJSONString(string(prices))) + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + info := &relaycommon.RelayInfo{OriginModelName: "requested-model", ExecutionModelName: "cheap-model", UserGroup: "default", UsingGroup: "default"} + priceData, err := ModelPriceHelper(ctx, info, 100, &types.TokenCountMeta{}) + require.NoError(t, err) + require.Equal(t, 0.25, priceData.ModelPrice) + require.Equal(t, "requested-model", info.OriginModelName) +} diff --git a/router/api-router.go b/router/api-router.go index 31c595e00db2..c4ef77f3c4a1 100644 --- a/router/api-router.go +++ b/router/api-router.go @@ -203,6 +203,19 @@ func SetApiRouter(router *gin.Engine) { optionRoute.POST("/waffo-pancake/subscription-product", controller.CreateWaffoPancakeSubscriptionProduct) optionRoute.GET("/waffo-pancake/subscription-product-options", controller.ListWaffoPancakeSubscriptionProductOptions) } + intelligentRoutingRoute := apiRouter.Group("/intelligent-routing") + intelligentRoutingRoute.Use(middleware.RootAuth()) + { + intelligentRoutingRoute.GET("/policies", controller.ListIntelligentRoutingPolicies) + intelligentRoutingRoute.GET("/policies/:id", controller.GetIntelligentRoutingPolicy) + intelligentRoutingRoute.POST("/policies", controller.CreateIntelligentRoutingPolicy) + intelligentRoutingRoute.PUT("/policies/:id", controller.UpdateIntelligentRoutingPolicy) + intelligentRoutingRoute.POST("/policies/:id/validate", controller.ValidateIntelligentRoutingPolicy) + intelligentRoutingRoute.POST("/policies/:id/publish", controller.PublishIntelligentRoutingPolicy) + intelligentRoutingRoute.POST("/policies/versions/:version/rollback", controller.RollbackIntelligentRoutingPolicy) + intelligentRoutingRoute.GET("/rollout", controller.GetIntelligentRoutingRollout) + intelligentRoutingRoute.PUT("/rollout", controller.UpdateIntelligentRoutingRollout) + } // Custom OAuth provider management (root only) customOAuthRoute := apiRouter.Group("/custom-oauth-provider") diff --git a/router/intelligent_routing_routes_test.go b/router/intelligent_routing_routes_test.go new file mode 100644 index 000000000000..453d7d625cd1 --- /dev/null +++ b/router/intelligent_routing_routes_test.go @@ -0,0 +1,35 @@ +package router + +import ( + "net/http" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" +) + +func TestIntelligentRoutingAdminRoutes(t *testing.T) { + gin.SetMode(gin.TestMode) + engine := gin.New() + SetApiRouter(engine) + routes := make(map[string]struct{}, len(engine.Routes())) + for _, route := range engine.Routes() { + routes[route.Method+" "+route.Path] = struct{}{} + } + + expected := []string{ + http.MethodGet + " /api/intelligent-routing/policies", + http.MethodGet + " /api/intelligent-routing/policies/:id", + http.MethodPost + " /api/intelligent-routing/policies", + http.MethodPut + " /api/intelligent-routing/policies/:id", + http.MethodPost + " /api/intelligent-routing/policies/:id/validate", + http.MethodPost + " /api/intelligent-routing/policies/:id/publish", + http.MethodPost + " /api/intelligent-routing/policies/versions/:version/rollback", + http.MethodGet + " /api/intelligent-routing/rollout", + http.MethodPut + " /api/intelligent-routing/rollout", + } + for _, route := range expected { + _, ok := routes[route] + assert.True(t, ok, route) + } +} diff --git a/service/billing_session.go b/service/billing_session.go index afc706a7a1a5..55abd34e4552 100644 --- a/service/billing_session.go +++ b/service/billing_session.go @@ -401,7 +401,7 @@ func NewBillingSession(c *gin.Context, relayInfo *relaycommon.RelayInfo, preCons funding: &SubscriptionFunding{ requestId: relayInfo.RequestId, userId: relayInfo.UserId, - modelName: relayInfo.OriginModelName, + modelName: relayInfo.GetBillingModelName(), amount: subConsume, }, } diff --git a/service/channel_affinity_usage_cache_test.go b/service/channel_affinity_usage_cache_test.go index 876297b21c05..7d22d58e4ab6 100644 --- a/service/channel_affinity_usage_cache_test.go +++ b/service/channel_affinity_usage_cache_test.go @@ -3,8 +3,8 @@ package service import ( "fmt" "net/http/httptest" + "sync/atomic" "testing" - "time" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/types" @@ -12,6 +12,8 @@ import ( "github.com/stretchr/testify/require" ) +var channelAffinityStatsTestSequence atomic.Uint64 + func buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP string) *gin.Context { rec := httptest.NewRecorder() ctx, _ := gin.CreateTestContext(rec) @@ -26,9 +28,10 @@ func buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP string) } func TestObserveChannelAffinityUsageCacheByRelayFormat_ClaudeMode(t *testing.T) { - ruleName := fmt.Sprintf("rule_%d", time.Now().UnixNano()) + testID := channelAffinityStatsTestSequence.Add(1) + ruleName := fmt.Sprintf("rule_%d", testID) usingGroup := "default" - keyFP := fmt.Sprintf("fp_%d", time.Now().UnixNano()) + keyFP := fmt.Sprintf("fp_%d", testID) ctx := buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP) usage := &dto.Usage{ @@ -53,9 +56,10 @@ func TestObserveChannelAffinityUsageCacheByRelayFormat_ClaudeMode(t *testing.T) } func TestObserveChannelAffinityUsageCacheByRelayFormat_MixedMode(t *testing.T) { - ruleName := fmt.Sprintf("rule_%d", time.Now().UnixNano()) + testID := channelAffinityStatsTestSequence.Add(1) + ruleName := fmt.Sprintf("rule_%d", testID) usingGroup := "default" - keyFP := fmt.Sprintf("fp_%d", time.Now().UnixNano()) + keyFP := fmt.Sprintf("fp_%d", testID) ctx := buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP) openAIUsage := &dto.Usage{ @@ -83,9 +87,10 @@ func TestObserveChannelAffinityUsageCacheByRelayFormat_MixedMode(t *testing.T) { } func TestObserveChannelAffinityUsageCacheByRelayFormat_UnsupportedModeKeepsEmpty(t *testing.T) { - ruleName := fmt.Sprintf("rule_%d", time.Now().UnixNano()) + testID := channelAffinityStatsTestSequence.Add(1) + ruleName := fmt.Sprintf("rule_%d", testID) usingGroup := "default" - keyFP := fmt.Sprintf("fp_%d", time.Now().UnixNano()) + keyFP := fmt.Sprintf("fp_%d", testID) ctx := buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP) usage := &dto.Usage{ diff --git a/service/intelligent_routing/budget.go b/service/intelligent_routing/budget.go new file mode 100644 index 000000000000..119b8901fc9c --- /dev/null +++ b/service/intelligent_routing/budget.go @@ -0,0 +1,50 @@ +package intelligent_routing + +import ( + "time" + + hosttypes "github.com/QuantumNous/new-api/types" + "github.com/shopspring/decimal" +) + +type ExecutionBudget struct { + startedAt time.Time + duration time.Duration + maxCost decimal.Decimal + spent decimal.Decimal + finalUsed bool +} + +func NewExecutionBudget(nodes []hosttypes.IntelligentRouteNode, multiplier float64, duration time.Duration, now time.Time) *ExecutionBudget { + maxCost := decimal.Zero + if len(nodes) > 0 { + maxCost = nodes[0].ExpectedCost.Mul(decimal.NewFromFloat(multiplier)) + } + return &ExecutionBudget{startedAt: now, duration: duration, maxCost: maxCost} +} + +func (budget *ExecutionBudget) SelectAttempt(nodes []hosttypes.IntelligentRouteNode, requestedIndex int, now time.Time) (int, bool) { + if budget == nil || requestedIndex < 0 || requestedIndex >= len(nodes) { + return 0, false + } + finalIndex := len(nodes) - 1 + withinTime := budget.duration <= 0 || now.Sub(budget.startedAt) <= budget.duration + withinCost := budget.spent.Add(nodes[requestedIndex].ExpectedCost).LessThanOrEqual(budget.maxCost) + if withinTime && withinCost { + if requestedIndex == finalIndex { + budget.finalUsed = true + } + return requestedIndex, true + } + if budget.finalUsed { + return 0, false + } + budget.finalUsed = true + return finalIndex, true +} + +func (budget *ExecutionBudget) Record(node hosttypes.IntelligentRouteNode) { + if budget != nil { + budget.spent = budget.spent.Add(node.ExpectedCost) + } +} diff --git a/service/intelligent_routing/budget_test.go b/service/intelligent_routing/budget_test.go new file mode 100644 index 000000000000..6cfa9a7f3c8e --- /dev/null +++ b/service/intelligent_routing/budget_test.go @@ -0,0 +1,43 @@ +package intelligent_routing + +import ( + "testing" + "time" + + hosttypes "github.com/QuantumNous/new-api/types" + "github.com/shopspring/decimal" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExecutionBudgetJumpsToFinalCandidateWhenCostWouldBeExceeded(t *testing.T) { + now := time.Unix(1000, 0) + nodes := []hosttypes.IntelligentRouteNode{ + {Model: "first", ExpectedCost: decimal.NewFromInt(1)}, + {Model: "second", ExpectedCost: decimal.NewFromInt(2)}, + {Model: "final", ExpectedCost: decimal.NewFromInt(5)}, + } + budget := NewExecutionBudget(nodes, 2.5, 30*time.Second, now) + index, ok := budget.SelectAttempt(nodes, 0, now) + require.True(t, ok) + assert.Equal(t, 0, index) + budget.Record(nodes[index]) + index, ok = budget.SelectAttempt(nodes, 1, now.Add(time.Second)) + require.True(t, ok) + assert.Equal(t, 2, index) + budget.Record(nodes[index]) + _, ok = budget.SelectAttempt(nodes, 2, now.Add(2*time.Second)) + assert.False(t, ok) +} + +func TestExecutionBudgetAllowsOnlyFinalCandidateAfterDeadline(t *testing.T) { + now := time.Unix(1000, 0) + nodes := []hosttypes.IntelligentRouteNode{{ExpectedCost: decimal.NewFromInt(1)}, {ExpectedCost: decimal.NewFromInt(1)}, {ExpectedCost: decimal.NewFromInt(1)}} + budget := NewExecutionBudget(nodes, 2.5, 30*time.Second, now) + index, ok := budget.SelectAttempt(nodes, 0, now) + require.True(t, ok) + budget.Record(nodes[index]) + index, ok = budget.SelectAttempt(nodes, 1, now.Add(31*time.Second)) + require.True(t, ok) + assert.Equal(t, 2, index) +} diff --git a/service/intelligent_routing/catalog.go b/service/intelligent_routing/catalog.go new file mode 100644 index 000000000000..73ec712fc62c --- /dev/null +++ b/service/intelligent_routing/catalog.go @@ -0,0 +1,99 @@ +package intelligent_routing + +import ( + "strings" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" +) + +type Candidate struct { + Model string + ChannelID int + Tier int + InputPrice float64 + OutputPrice float64 + ContextLimit int + Capabilities map[Capability]bool + ResponseTimeMS int + PredictedSuccess float64 + FailureRate float64 + HealthTier int +} + +type ChannelSource func(group, requestPath string) []*model.Channel + +type Catalog struct { + config routingsetting.Config + source ChannelSource + health *HealthTracker + now func() time.Time +} + +func NewCatalog(config routingsetting.Config, source ChannelSource) Catalog { + if source == nil { + source = model.ListEnabledChannelsForRouting + } + return NewCatalogWithHealth(config, source, &DefaultHealthTracker, time.Now) +} + +func NewCatalogWithHealth(config routingsetting.Config, source ChannelSource, health *HealthTracker, now func() time.Time) Catalog { + if source == nil { + source = model.ListEnabledChannelsForRouting + } + return Catalog{config: config, source: source, health: health, now: now} +} + +func (catalog Catalog) Build(group, requestPath string) []Candidate { + policies := make(map[string]routingsetting.ModelPolicy, len(catalog.config.Models)) + for _, policy := range catalog.config.Models { + if policy.InputPrice == 0 && policy.OutputPrice == 0 { + continue + } + policies[policy.Model] = policy + } + var candidates []Candidate + for _, channel := range catalog.source(group, requestPath) { + if channel == nil || channel.Status != common.ChannelStatusEnabled || !contains(channel.GetGroups(), group) { + continue + } + for _, modelName := range channel.GetModels() { + health := catalog.health.SnapshotAt(channel.Id, catalog.now()) + if health.Tier == HealthOpen { + continue + } + modelName = strings.TrimSpace(modelName) + policy, ok := policies[modelName] + if !ok { + continue + } + capabilities := make(map[Capability]bool, len(policy.Capabilities)) + for _, capability := range policy.Capabilities { + capabilities[Capability(capability)] = true + } + candidates = append(candidates, Candidate{ + Model: modelName, ChannelID: channel.Id, Tier: policy.Tier, + InputPrice: policy.InputPrice, OutputPrice: policy.OutputPrice, + ContextLimit: policy.ContextLimit, Capabilities: capabilities, + ResponseTimeMS: channel.ResponseTime, PredictedSuccess: coldStartQualityPrior(policy.Tier), + FailureRate: health.FailureRate, HealthTier: health.Tier, + }) + } + } + return candidates +} + +func coldStartQualityPrior(tier int) float64 { + return [...]float64{.88, .92, .96, .99}[tier] +} + +func contains(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} diff --git a/service/intelligent_routing/catalog_test.go b/service/intelligent_routing/catalog_test.go new file mode 100644 index 000000000000..7fb6dec91c18 --- /dev/null +++ b/service/intelligent_routing/catalog_test.go @@ -0,0 +1,71 @@ +package intelligent_routing + +import ( + "testing" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCatalogExcludesUnpricedDisabledAndUnsupportedCandidates(t *testing.T) { + config := routingsetting.Config{Models: []routingsetting.ModelPolicy{ + {Model: "cheap", Tier: 0, InputPrice: 1, OutputPrice: 2, ContextLimit: 8192, Capabilities: []string{"tools"}}, + {Model: "unpriced", Tier: 1, ContextLimit: 8192}, + }} + channels := []*model.Channel{ + {Id: 7, Status: common.ChannelStatusEnabled, Models: "cheap", Group: "default"}, + {Id: 8, Status: common.ChannelStatusManuallyDisabled, Models: "cheap", Group: "default"}, + {Id: 9, Status: common.ChannelStatusEnabled, Models: "unpriced", Group: "default"}, + } + got := NewCatalog(config, func(string, string) []*model.Channel { return channels }).Build("default", "/v1/chat/completions") + require.Len(t, got, 1) + assert.Equal(t, "cheap", got[0].Model) + assert.Equal(t, 7, got[0].ChannelID) + assert.True(t, got[0].Capabilities[CapabilityTools]) +} + +func TestCatalogReturnsIndependentCandidateValues(t *testing.T) { + config := routingsetting.Config{Models: []routingsetting.ModelPolicy{{Model: "cheap", Tier: 0, InputPrice: 1, OutputPrice: 2}}} + channel := &model.Channel{Id: 7, Status: common.ChannelStatusEnabled, Models: "cheap", Group: "default", ResponseTime: 20} + catalog := NewCatalog(config, func(string, string) []*model.Channel { return []*model.Channel{channel} }) + first := catalog.Build("default", "/v1/chat/completions") + channel.ResponseTime = 900 + assert.Equal(t, 20, first[0].ResponseTimeMS) +} + +func TestCatalogExcludesChannelsWithOpenHealthCircuit(t *testing.T) { + config := routingsetting.Config{Models: []routingsetting.ModelPolicy{{Model: "cheap", InputPrice: 1, OutputPrice: 2}}} + channels := []*model.Channel{ + {Id: 7, Status: common.ChannelStatusEnabled, Models: "cheap", Group: "default"}, + {Id: 8, Status: common.ChannelStatusEnabled, Models: "cheap", Group: "default"}, + } + var health HealthTracker + now := time.Unix(1000, 0) + for i := 0; i < 20; i++ { + health.RecordAt(7, false, now) + health.RecordAt(8, true, now) + } + catalog := NewCatalogWithHealth(config, func(string, string) []*model.Channel { return channels }, &health, func() time.Time { return now }) + got := catalog.Build("default", "/v1/chat/completions") + require.Len(t, got, 1) + assert.Equal(t, 8, got[0].ChannelID) + assert.Equal(t, HealthHealthy, got[0].HealthTier) +} + +func TestCatalogAssignsConservativeColdStartQualityPriorByTier(t *testing.T) { + config := routingsetting.Config{Models: []routingsetting.ModelPolicy{ + {Model: "l0", Tier: 0, InputPrice: 1}, {Model: "l1", Tier: 1, InputPrice: 1}, + {Model: "l2", Tier: 2, InputPrice: 1}, {Model: "l3", Tier: 3, InputPrice: 1}, + }} + channel := &model.Channel{Id: 7, Status: common.ChannelStatusEnabled, Models: "l0,l1,l2,l3", Group: "default"} + got := NewCatalog(config, func(string, string) []*model.Channel { return []*model.Channel{channel} }).Build("default", "/v1/chat/completions") + require.Len(t, got, 4) + assert.InDelta(t, .88, got[0].PredictedSuccess, .0001) + assert.InDelta(t, .92, got[1].PredictedSuccess, .0001) + assert.InDelta(t, .96, got[2].PredictedSuccess, .0001) + assert.InDelta(t, .99, got[3].PredictedSuccess, .0001) +} diff --git a/service/intelligent_routing/features.go b/service/intelligent_routing/features.go new file mode 100644 index 000000000000..c70206c77611 --- /dev/null +++ b/service/intelligent_routing/features.go @@ -0,0 +1,175 @@ +package intelligent_routing + +import ( + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/types" + routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" +) + +type TaskType = routingsetting.TaskType + +const ( + TaskTranslation = routingsetting.TaskTranslation + TaskSummary = routingsetting.TaskSummary + TaskGeneral = routingsetting.TaskGeneral + TaskExtraction = routingsetting.TaskExtraction + TaskCode = routingsetting.TaskCode + TaskReasoning = routingsetting.TaskReasoning + TaskJSON = routingsetting.TaskJSON + TaskTool = routingsetting.TaskTool +) + +type Capability string + +const ( + CapabilityTools Capability = "tools" + CapabilityJSONSchema Capability = "json_schema" + CapabilityVision Capability = "vision" + CapabilityAudio Capability = "audio" +) + +type Input struct { + Request dto.Request + RelayFormat types.RelayFormat + PromptTokens int + RequestPath string +} + +type Features struct { + Task TaskType + PromptTokens int + MaxOutputTokens int + HasTools bool + RequiresJSONSchema bool + HasImage bool + HasAudio bool + IsStream bool + MinimumTier int +} + +type Requirements struct { + Capabilities map[Capability]bool + MinimumTier int + ContextNeeded int +} + +func ExtractFeatures(input Input) Features { + features := Features{Task: TaskGeneral, PromptTokens: input.PromptTokens, MinimumTier: 1} + var text string + switch request := input.Request.(type) { + case *dto.GeneralOpenAIRequest: + features.HasTools = len(request.Tools) > 0 || len(request.Functions) > 0 + features.RequiresJSONSchema = request.ResponseFormat != nil && request.ResponseFormat.Type == "json_schema" + features.IsStream = request.Stream != nil && *request.Stream + features.MaxOutputTokens = int(request.GetMaxTokens()) + for _, message := range request.Messages { + switch content := message.Content.(type) { + case string: + text += " " + content + case []dto.MediaContent: + for _, item := range content { + text += " " + item.Text + features.HasImage = features.HasImage || item.ImageUrl != nil + features.HasAudio = features.HasAudio || item.InputAudio != nil + } + } + } + if request.ReasoningEffort != "" { + features.Task, features.MinimumTier = TaskReasoning, 2 + } + case *dto.OpenAIResponsesRequest: + features.HasTools = len(request.Tools) > 0 + features.RequiresJSONSchema = strings.Contains(string(request.Text), "json_schema") + features.IsStream = request.Stream != nil && *request.Stream + if request.MaxOutputTokens != nil { + features.MaxOutputTokens = int(*request.MaxOutputTokens) + } + text = string(request.Input) + " " + string(request.Instructions) + case *dto.ClaudeRequest: + features.HasTools = request.Tools != nil + features.RequiresJSONSchema = len(request.OutputFormat) > 0 + features.IsStream = request.Stream != nil && *request.Stream + if request.MaxTokens != nil { + features.MaxOutputTokens = int(*request.MaxTokens) + } + text = request.Prompt + } + if features.HasTools { + features.Task, features.MinimumTier = TaskTool, 2 + } else if features.RequiresJSONSchema { + features.Task, features.MinimumTier = TaskJSON, 2 + } else if features.Task == TaskGeneral { + features.Task, features.MinimumTier = classifyText(text) + } + return features +} + +func DeriveRequirements(features Features) Requirements { + requirements := Requirements{ + Capabilities: make(map[Capability]bool), + MinimumTier: features.MinimumTier, + ContextNeeded: features.PromptTokens + features.MaxOutputTokens, + } + if features.HasTools { + requirements.Capabilities[CapabilityTools] = true + } + if features.RequiresJSONSchema { + requirements.Capabilities[CapabilityJSONSchema] = true + } + if features.HasImage { + requirements.Capabilities[CapabilityVision] = true + } + if features.HasAudio { + requirements.Capabilities[CapabilityAudio] = true + } + return requirements +} + +func ConversationSeed(request dto.Request) string { + switch value := request.(type) { + case *dto.GeneralOpenAIRequest: + for _, message := range value.Messages { + if message.Role != "user" { + continue + } + switch content := message.Content.(type) { + case string: + return strings.TrimSpace(content) + case []dto.MediaContent: + for _, item := range content { + if text := strings.TrimSpace(item.Text); text != "" { + return text + } + } + } + } + case *dto.OpenAIResponsesRequest: + return strings.TrimSpace(string(value.Input)) + } + return "" +} + +func classifyText(text string) (TaskType, int) { + lower := strings.ToLower(text) + checks := []struct { + words []string + task TaskType + tier int + }{ + {[]string{"translate", "翻译"}, TaskTranslation, 0}, + {[]string{"summarize", "summary", "总结", "摘要"}, TaskSummary, 1}, + {[]string{"extract", "提取", "分类"}, TaskExtraction, 0}, + {[]string{"write a go", "write code", "代码", "function", "debug"}, TaskCode, 2}, + {[]string{"prove", "推理", "数学", "calculate"}, TaskReasoning, 2}, + } + for _, check := range checks { + for _, word := range check.words { + if strings.Contains(lower, word) { + return check.task, check.tier + } + } + } + return TaskGeneral, 1 +} diff --git a/service/intelligent_routing/features_test.go b/service/intelligent_routing/features_test.go new file mode 100644 index 000000000000..0ad24319b69f --- /dev/null +++ b/service/intelligent_routing/features_test.go @@ -0,0 +1,57 @@ +package intelligent_routing + +import ( + "testing" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/types" + "github.com/stretchr/testify/assert" +) + +func TestExtractFeaturesOpenAIRequests(t *testing.T) { + tests := []struct { + name string + request *dto.GeneralOpenAIRequest + task TaskType + tier int + tools bool + json bool + }{ + {name: "translation", request: requestWithText("Translate this sentence into Chinese"), task: TaskTranslation, tier: 0}, + {name: "summary", request: requestWithText("Summarize this article"), task: TaskSummary, tier: 1}, + {name: "code", request: requestWithText("Write a Go function that parses a request"), task: TaskCode, tier: 2}, + {name: "tool", request: &dto.GeneralOpenAIRequest{Messages: []dto.Message{{Role: "user", Content: "weather"}}, Tools: []dto.ToolCallRequest{{Type: "function"}}}, task: TaskTool, tier: 2, tools: true}, + {name: "schema", request: &dto.GeneralOpenAIRequest{Messages: []dto.Message{{Role: "user", Content: "extract fields"}}, ResponseFormat: &dto.ResponseFormat{Type: "json_schema"}}, task: TaskJSON, tier: 2, json: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ExtractFeatures(Input{Request: tt.request, RelayFormat: types.RelayFormatOpenAI, PromptTokens: 120}) + assert.Equal(t, tt.task, got.Task) + assert.Equal(t, tt.tier, got.MinimumTier) + assert.Equal(t, tt.tools, got.HasTools) + assert.Equal(t, tt.json, got.RequiresJSONSchema) + }) + } +} + +func TestDeriveRequirementsCarriesProtocolConstraints(t *testing.T) { + features := Features{MinimumTier: 2, PromptTokens: 8000, MaxOutputTokens: 2000, HasTools: true, HasImage: true} + got := DeriveRequirements(features) + assert.Equal(t, 2, got.MinimumTier) + assert.Equal(t, 10000, got.ContextNeeded) + assert.True(t, got.Capabilities[CapabilityTools]) + assert.True(t, got.Capabilities[CapabilityVision]) +} + +func TestConversationSeedUsesFirstUserMessage(t *testing.T) { + request := &dto.GeneralOpenAIRequest{Messages: []dto.Message{ + {Role: "system", Content: "be concise"}, + {Role: "user", Content: "first question"}, + {Role: "user", Content: "later question"}, + }} + assert.Equal(t, "first question", ConversationSeed(request)) +} + +func requestWithText(text string) *dto.GeneralOpenAIRequest { + return &dto.GeneralOpenAIRequest{Messages: []dto.Message{{Role: "user", Content: text}}} +} diff --git a/service/intelligent_routing/health.go b/service/intelligent_routing/health.go new file mode 100644 index 000000000000..f09ba02fe79c --- /dev/null +++ b/service/intelligent_routing/health.go @@ -0,0 +1,83 @@ +package intelligent_routing + +import ( + "sync" + "time" +) + +const ( + HealthHealthy = iota + HealthDegraded + HealthProbation + HealthOpen +) + +type HealthSnapshot struct { + Tier int + FailureRate float64 +} + +type healthEvent struct { + at time.Time + success bool +} + +type HealthTracker struct { + mu sync.Mutex + events map[int][]healthEvent +} + +var DefaultHealthTracker HealthTracker + +func (tracker *HealthTracker) RecordAt(channelID int, success bool, now time.Time) { + tracker.mu.Lock() + defer tracker.mu.Unlock() + if tracker.events == nil { + tracker.events = make(map[int][]healthEvent) + } + tracker.events[channelID] = append(recentHealthEvents(tracker.events[channelID], now), healthEvent{at: now, success: success}) +} + +func (tracker *HealthTracker) Record(channelID int, success bool) { + tracker.RecordAt(channelID, success, time.Now()) +} + +func (tracker *HealthTracker) SnapshotAt(channelID int, now time.Time) HealthSnapshot { + tracker.mu.Lock() + defer tracker.mu.Unlock() + events := recentHealthEvents(tracker.events[channelID], now) + if tracker.events != nil { + tracker.events[channelID] = events + } + if len(events) < 20 { + return HealthSnapshot{Tier: HealthProbation} + } + failures := 0 + for _, event := range events { + if !event.success { + failures++ + } + } + failureRate := float64(failures) / float64(len(events)) + tier := HealthHealthy + if failureRate >= .05 { + tier = HealthDegraded + } + if failureRate > .05 { + tier = HealthOpen + } + return HealthSnapshot{Tier: tier, FailureRate: failureRate} +} + +func (tracker *HealthTracker) Snapshot(channelID int) HealthSnapshot { + return tracker.SnapshotAt(channelID, time.Now()) +} + +func recentHealthEvents(events []healthEvent, now time.Time) []healthEvent { + cutoff := now.Add(-time.Minute) + first := 0 + for first < len(events) && events[first].at.Before(cutoff) { + first++ + } + return events[first:] +} diff --git a/service/intelligent_routing/health_test.go b/service/intelligent_routing/health_test.go new file mode 100644 index 000000000000..12a1bd1f147f --- /dev/null +++ b/service/intelligent_routing/health_test.go @@ -0,0 +1,41 @@ +package intelligent_routing + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestHealthTrackerClassifiesAndExpiresRollingWindow(t *testing.T) { + var tracker HealthTracker + now := time.Unix(1000, 0) + for i := 0; i < 20; i++ { + tracker.RecordAt(7, i != 0, now) + } + got := tracker.SnapshotAt(7, now) + assert.Equal(t, HealthDegraded, got.Tier) + assert.InDelta(t, .05, got.FailureRate, .0001) + tracker.RecordAt(7, true, now.Add(61*time.Second)) + got = tracker.SnapshotAt(7, now.Add(61*time.Second)) + assert.Equal(t, HealthProbation, got.Tier) + assert.Zero(t, got.FailureRate) +} + +func TestHealthTrackerOpensCircuitForRepeatedFailures(t *testing.T) { + var tracker HealthTracker + now := time.Unix(1000, 0) + for i := 0; i < 20; i++ { + tracker.RecordAt(9, false, now) + } + assert.Equal(t, HealthOpen, tracker.SnapshotAt(9, now).Tier) +} + +func TestHealthTrackerOpensBelowNinetyFivePercentSuccess(t *testing.T) { + var tracker HealthTracker + now := time.Unix(1000, 0) + for i := 0; i < 20; i++ { + tracker.RecordAt(11, i >= 2, now) + } + assert.Equal(t, HealthOpen, tracker.SnapshotAt(11, now).Tier) +} diff --git a/service/intelligent_routing/metrics.go b/service/intelligent_routing/metrics.go new file mode 100644 index 000000000000..18156f7e7fcf --- /dev/null +++ b/service/intelligent_routing/metrics.go @@ -0,0 +1,62 @@ +package intelligent_routing + +import ( + "sync" + "time" +) + +type Observation struct { + NoRoute bool + CandidateTier int + ExpectedSaving float64 + PlanningDuration time.Duration +} + +type MetricsSnapshot struct { + Planned int64 + NoRoute int64 + ByTier map[int]int64 + AverageExpectedSaving float64 + AveragePlanningDuration time.Duration +} + +type Metrics struct { + mu sync.Mutex + planned int64 + noRoute int64 + byTier map[int]int64 + totalSaving float64 + totalDuration time.Duration +} + +var DefaultMetrics Metrics + +func (metrics *Metrics) Observe(observation Observation) { + metrics.mu.Lock() + defer metrics.mu.Unlock() + metrics.planned++ + if observation.NoRoute { + metrics.noRoute++ + } else { + if metrics.byTier == nil { + metrics.byTier = make(map[int]int64) + } + metrics.byTier[observation.CandidateTier]++ + } + metrics.totalSaving += observation.ExpectedSaving + metrics.totalDuration += observation.PlanningDuration +} + +func (metrics *Metrics) Snapshot() MetricsSnapshot { + metrics.mu.Lock() + defer metrics.mu.Unlock() + snapshot := MetricsSnapshot{Planned: metrics.planned, NoRoute: metrics.noRoute, ByTier: make(map[int]int64, len(metrics.byTier))} + for tier, count := range metrics.byTier { + snapshot.ByTier[tier] = count + } + if metrics.planned > 0 { + snapshot.AverageExpectedSaving = metrics.totalSaving / float64(metrics.planned) + snapshot.AveragePlanningDuration = metrics.totalDuration / time.Duration(metrics.planned) + } + return snapshot +} diff --git a/service/intelligent_routing/metrics_test.go b/service/intelligent_routing/metrics_test.go new file mode 100644 index 000000000000..091fdc8f85b5 --- /dev/null +++ b/service/intelligent_routing/metrics_test.go @@ -0,0 +1,20 @@ +package intelligent_routing + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestMetricsObserveAggregatesRoutingOutcomes(t *testing.T) { + var metrics Metrics + metrics.Observe(Observation{CandidateTier: 1, ExpectedSaving: .4, PlanningDuration: 3 * time.Millisecond}) + metrics.Observe(Observation{NoRoute: true, PlanningDuration: time.Millisecond}) + got := metrics.Snapshot() + assert.EqualValues(t, 2, got.Planned) + assert.EqualValues(t, 1, got.NoRoute) + assert.EqualValues(t, 1, got.ByTier[1]) + assert.InDelta(t, .2, got.AverageExpectedSaving, .0001) + assert.Equal(t, 2*time.Millisecond, got.AveragePlanningDuration) +} diff --git a/service/intelligent_routing/planner.go b/service/intelligent_routing/planner.go new file mode 100644 index 000000000000..3245a45b390b --- /dev/null +++ b/service/intelligent_routing/planner.go @@ -0,0 +1,178 @@ +package intelligent_routing + +import ( + "errors" + "sort" + + hosttypes "github.com/QuantumNous/new-api/types" + "github.com/shopspring/decimal" +) + +type PlanInput struct { + RequestedModel string + PolicyVersion int + Features Features + Requirements Requirements + Candidates []Candidate + QualityThreshold float64 + MaxAttempts int + MaxEndpointsPerModel int + MaxCostMultiplier float64 + PreferredModel string + PreferredChannelID int +} + +type RouteNode = hosttypes.IntelligentRouteNode +type RoutePlan = hosttypes.IntelligentRoutePlan + +func Plan(input PlanInput) (RoutePlan, error) { + if input.MaxAttempts < 1 || input.MaxEndpointsPerModel < 1 || input.MaxCostMultiplier < 1 { + return RoutePlan{}, errors.New("invalid route plan budget") + } + eligible := make([]Candidate, 0, len(input.Candidates)) + for _, candidate := range input.Candidates { + if candidate.Tier < input.Requirements.MinimumTier || !supports(candidate, input.Requirements.Capabilities) { + continue + } + if candidate.ContextLimit > 0 && input.Requirements.ContextNeeded*10 > candidate.ContextLimit*7 { + continue + } + eligible = append(eligible, candidate) + } + if len(eligible) == 0 { + return RoutePlan{}, errors.New("no eligible intelligent routing candidate") + } + qualified := make([]Candidate, 0, len(eligible)) + for _, candidate := range eligible { + if candidate.PredictedSuccess >= input.QualityThreshold { + qualified = append(qualified, candidate) + } + } + fallbackByQuality := len(qualified) == 0 + if fallbackByQuality { + qualified = eligible + sort.SliceStable(qualified, func(i, j int) bool { + if qualified[i].PredictedSuccess != qualified[j].PredictedSuccess { + return qualified[i].PredictedSuccess > qualified[j].PredictedSuccess + } + return expectedCost(input.Features, qualified[i]).LessThan(expectedCost(input.Features, qualified[j])) + }) + } else { + sort.SliceStable(qualified, func(i, j int) bool { + left, right := qualified[i], qualified[j] + if left.HealthTier != right.HealthTier { + return left.HealthTier < right.HealthTier + } + leftCost, rightCost := expectedCost(input.Features, left), expectedCost(input.Features, right) + if !leftCost.Equal(rightCost) { + return leftCost.LessThan(rightCost) + } + if left.ResponseTimeMS != right.ResponseTimeMS { + return left.ResponseTimeMS < right.ResponseTimeMS + } + if left.FailureRate != right.FailureRate { + return left.FailureRate < right.FailureRate + } + return left.ChannelID < right.ChannelID + }) + } + if !fallbackByQuality && input.PreferredModel != "" && len(qualified) > 1 { + cheapestCost := expectedCost(input.Features, qualified[0]) + for _, candidate := range qualified[1:] { + cost := expectedCost(input.Features, candidate) + if cost.LessThan(cheapestCost) { + cheapestCost = cost + } + } + limit := cheapestCost.Mul(decimal.NewFromFloat(1.15)) + for i, candidate := range qualified { + if candidate.Model == input.PreferredModel && candidate.ChannelID == input.PreferredChannelID && candidate.HealthTier != HealthDegraded && candidate.HealthTier != HealthOpen && !expectedCost(input.Features, candidate).GreaterThan(limit) { + copy(qualified[1:i+1], qualified[0:i]) + qualified[0] = candidate + break + } + } + } + if fallbackByQuality { + nodes := make([]RouteNode, 0, min(input.MaxAttempts, len(qualified))) + perModel := make(map[string]int) + seen := make(map[[2]interface{}]struct{}) + for _, candidate := range qualified { + key := [2]interface{}{candidate.Model, candidate.ChannelID} + if _, ok := seen[key]; ok || perModel[candidate.Model] >= input.MaxEndpointsPerModel { + continue + } + nodes = append(nodes, routeNode(input.Features, candidate)) + seen[key] = struct{}{} + perModel[candidate.Model]++ + if len(nodes) == input.MaxAttempts { + break + } + } + if len(nodes) == 0 { + return RoutePlan{}, errors.New("route plan is empty") + } + return RoutePlan{RequestedModel: input.RequestedModel, PolicyVersion: input.PolicyVersion, Nodes: nodes, MaxAttempts: input.MaxAttempts, MaxCostMultiplier: input.MaxCostMultiplier}, nil + } + strongest := len(qualified) - 1 + for i := 0; i < len(qualified)-1; i++ { + if qualified[i].PredictedSuccess > qualified[strongest].PredictedSuccess { + strongest = i + } + } + nodes := make([]RouteNode, 0, min(input.MaxAttempts, len(qualified))) + perModel := make(map[string]int) + seen := make(map[[2]interface{}]struct{}) + for i, candidate := range qualified { + if i == strongest || len(nodes) == input.MaxAttempts-1 { + continue + } + key := [2]interface{}{candidate.Model, candidate.ChannelID} + modelLimit := input.MaxEndpointsPerModel + if candidate.Model == qualified[strongest].Model { + modelLimit-- + } + if _, ok := seen[key]; ok || perModel[candidate.Model] >= modelLimit { + continue + } + nodes = append(nodes, routeNode(input.Features, candidate)) + seen[key] = struct{}{} + perModel[candidate.Model]++ + } + strongestCandidate := qualified[strongest] + strongestKey := [2]interface{}{strongestCandidate.Model, strongestCandidate.ChannelID} + if _, ok := seen[strongestKey]; !ok && perModel[strongestCandidate.Model] < input.MaxEndpointsPerModel { + nodes = append(nodes, routeNode(input.Features, strongestCandidate)) + } + if len(nodes) == 0 { + return RoutePlan{}, errors.New("route plan is empty") + } + return RoutePlan{RequestedModel: input.RequestedModel, PolicyVersion: input.PolicyVersion, Nodes: nodes, MaxAttempts: input.MaxAttempts, MaxCostMultiplier: input.MaxCostMultiplier}, nil +} + +func supports(candidate Candidate, required map[Capability]bool) bool { + for capability, needed := range required { + if needed && !candidate.Capabilities[capability] { + return false + } + } + return true +} + +func expectedCost(features Features, candidate Candidate) decimal.Decimal { + input := decimal.NewFromInt(int64(features.PromptTokens)).Mul(decimal.NewFromFloat(candidate.InputPrice)) + output := decimal.NewFromInt(int64(features.MaxOutputTokens)).Mul(decimal.NewFromFloat(candidate.OutputPrice)) + base := input.Add(output).Div(decimal.NewFromInt(1_000_000)) + failureRate := candidate.FailureRate + if failureRate < 0 { + failureRate = 0 + } + if failureRate > .99 { + failureRate = .99 + } + return base.Div(decimal.NewFromFloat(1 - failureRate)) +} + +func routeNode(features Features, candidate Candidate) RouteNode { + return RouteNode{Model: candidate.Model, ChannelID: candidate.ChannelID, Tier: candidate.Tier, PredictedSuccess: candidate.PredictedSuccess, ExpectedCost: expectedCost(features, candidate), ReasonCodes: []string{"quality_threshold_met", "lowest_expected_cost"}} +} diff --git a/service/intelligent_routing/planner_test.go b/service/intelligent_routing/planner_test.go new file mode 100644 index 000000000000..be557d01edec --- /dev/null +++ b/service/intelligent_routing/planner_test.go @@ -0,0 +1,140 @@ +package intelligent_routing + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPlanChoosesCheapestCandidateMeetingQualityThreshold(t *testing.T) { + got, err := Plan(PlanInput{ + RequestedModel: "client-model", PolicyVersion: 2, + Features: Features{Task: TaskGeneral, PromptTokens: 100, MaxOutputTokens: 50}, + Requirements: Requirements{Capabilities: map[Capability]bool{}, MinimumTier: 1, ContextNeeded: 150}, + Candidates: []Candidate{ + {Model: "cheap", ChannelID: 1, Tier: 1, InputPrice: 1, OutputPrice: 2, ContextLimit: 1000, PredictedSuccess: .92}, + {Model: "premium", ChannelID: 2, Tier: 3, InputPrice: 8, OutputPrice: 16, ContextLimit: 1000, PredictedSuccess: .99}, + }, + QualityThreshold: .90, MaxAttempts: 4, MaxEndpointsPerModel: 2, MaxCostMultiplier: 2.5, + }) + require.NoError(t, err) + require.Len(t, got.Nodes, 2) + assert.Equal(t, "cheap", got.Nodes[0].Model) + assert.Equal(t, "premium", got.Nodes[1].Model) + assert.Equal(t, 2, got.PolicyVersion) +} + +func TestPlanPrefersStickyRouteOnlyWithinFifteenPercentOfCheapest(t *testing.T) { + base := PlanInput{ + RequestedModel: "client-model", PolicyVersion: 2, + Features: Features{Task: TaskGeneral, PromptTokens: 1_000, MaxOutputTokens: 100}, + Requirements: Requirements{Capabilities: map[Capability]bool{}}, + QualityThreshold: .9, MaxAttempts: 3, MaxEndpointsPerModel: 2, MaxCostMultiplier: 2.5, + Candidates: []Candidate{ + {Model: "cheapest", ChannelID: 1, InputPrice: 1, OutputPrice: 1, PredictedSuccess: .95}, + {Model: "sticky", ChannelID: 2, InputPrice: 1.1, OutputPrice: 1.1, PredictedSuccess: .94}, + {Model: "safest", ChannelID: 3, InputPrice: 5, OutputPrice: 5, PredictedSuccess: .99}, + }, + PreferredModel: "sticky", PreferredChannelID: 2, + } + plan, err := Plan(base) + require.NoError(t, err) + assert.Equal(t, "sticky", plan.Nodes[0].Model) + + base.Candidates[1].InputPrice, base.Candidates[1].OutputPrice = 1.2, 1.2 + plan, err = Plan(base) + require.NoError(t, err) + assert.Equal(t, "cheapest", plan.Nodes[0].Model) + + base.Candidates[1].InputPrice, base.Candidates[1].OutputPrice = 1.1, 1.1 + base.Candidates[1].HealthTier = HealthDegraded + plan, err = Plan(base) + require.NoError(t, err) + assert.Equal(t, "cheapest", plan.Nodes[0].Model) +} + +func TestPlanDoesNotMoveCheapestFirstNodeWhenSuccessProbabilitiesTie(t *testing.T) { + plan, err := Plan(PlanInput{ + RequestedModel: "requested", Features: Features{PromptTokens: 100}, Requirements: Requirements{Capabilities: map[Capability]bool{}}, + QualityThreshold: .9, MaxAttempts: 3, MaxEndpointsPerModel: 2, MaxCostMultiplier: 2.5, + Candidates: []Candidate{ + {Model: "cheap", ChannelID: 1, InputPrice: 1, PredictedSuccess: .95}, + {Model: "middle", ChannelID: 2, InputPrice: 2, PredictedSuccess: .95}, + {Model: "expensive", ChannelID: 3, InputPrice: 3, PredictedSuccess: .95}, + }, + }) + require.NoError(t, err) + assert.Equal(t, "cheap", plan.Nodes[0].Model) +} + +func TestPlanReservesHighestSuccessCandidateEvenWhenOutsideCheapAttemptWindow(t *testing.T) { + plan, err := Plan(PlanInput{ + RequestedModel: "requested", Features: Features{PromptTokens: 100}, Requirements: Requirements{Capabilities: map[Capability]bool{}}, + QualityThreshold: .9, MaxAttempts: 4, MaxEndpointsPerModel: 2, MaxCostMultiplier: 2.5, + Candidates: []Candidate{ + {Model: "cheap-1", ChannelID: 1, InputPrice: 1, PredictedSuccess: .91}, + {Model: "cheap-2", ChannelID: 2, InputPrice: 2, PredictedSuccess: .92}, + {Model: "cheap-3", ChannelID: 3, InputPrice: 3, PredictedSuccess: .93}, + {Model: "cheap-4", ChannelID: 4, InputPrice: 4, PredictedSuccess: .94}, + {Model: "safest", ChannelID: 5, InputPrice: 100, PredictedSuccess: .99}, + }, + }) + require.NoError(t, err) + require.Len(t, plan.Nodes, 4) + assert.Equal(t, "safest", plan.Nodes[3].Model) +} + +func TestPlanIncludesEndpointFailureRiskInExpectedCost(t *testing.T) { + plan, err := Plan(PlanInput{ + Features: Features{PromptTokens: 1_000}, Requirements: Requirements{Capabilities: map[Capability]bool{}}, + QualityThreshold: .9, MaxAttempts: 3, MaxEndpointsPerModel: 2, MaxCostMultiplier: 2.5, + Candidates: []Candidate{ + {Model: "flaky", ChannelID: 1, InputPrice: 1, PredictedSuccess: .95, FailureRate: .5}, + {Model: "healthy", ChannelID: 2, InputPrice: 1.2, PredictedSuccess: .95}, + {Model: "safest", ChannelID: 3, InputPrice: 5, PredictedSuccess: .99}, + }, + }) + require.NoError(t, err) + assert.Equal(t, "healthy", plan.Nodes[0].Model) +} + +func TestPlanFiltersCapabilitiesAndContext(t *testing.T) { + got, err := Plan(PlanInput{ + Requirements: Requirements{Capabilities: map[Capability]bool{CapabilityTools: true}, MinimumTier: 2, ContextNeeded: 900}, + Candidates: []Candidate{ + {Model: "no-tools", ChannelID: 1, Tier: 2, ContextLimit: 2000, PredictedSuccess: .99, InputPrice: 1}, + {Model: "too-small", ChannelID: 2, Tier: 2, ContextLimit: 1000, PredictedSuccess: .99, InputPrice: 1, Capabilities: map[Capability]bool{CapabilityTools: true}}, + {Model: "eligible", ChannelID: 3, Tier: 2, ContextLimit: 2000, PredictedSuccess: .95, InputPrice: 2, Capabilities: map[Capability]bool{CapabilityTools: true}}, + }, QualityThreshold: .90, MaxAttempts: 4, MaxEndpointsPerModel: 2, MaxCostMultiplier: 2.5, + }) + require.NoError(t, err) + require.Len(t, got.Nodes, 1) + assert.Equal(t, "eligible", got.Nodes[0].Model) +} + +func TestPlanFallsBackToHighestSuccessWhenNoneMeetThreshold(t *testing.T) { + got, err := Plan(PlanInput{ + Requirements: Requirements{Capabilities: map[Capability]bool{}}, + Candidates: []Candidate{ + {Model: "cheap", ChannelID: 1, PredictedSuccess: .6, InputPrice: 1}, + {Model: "reliable", ChannelID: 2, PredictedSuccess: .8, InputPrice: 3}, + }, QualityThreshold: .9, MaxAttempts: 1, MaxEndpointsPerModel: 1, MaxCostMultiplier: 2.5, + }) + require.NoError(t, err) + require.Len(t, got.Nodes, 1) + assert.Equal(t, "reliable", got.Nodes[0].Model) +} + +func TestPlanOrdersAllFallbackCandidatesByDescendingSuccess(t *testing.T) { + got, err := Plan(PlanInput{ + Requirements: Requirements{Capabilities: map[Capability]bool{}}, + Candidates: []Candidate{ + {Model: "cheap", ChannelID: 1, PredictedSuccess: .6, InputPrice: 1}, + {Model: "reliable", ChannelID: 2, PredictedSuccess: .8, InputPrice: 3}, + }, QualityThreshold: .9, MaxAttempts: 2, MaxEndpointsPerModel: 1, MaxCostMultiplier: 2.5, + }) + require.NoError(t, err) + require.Len(t, got.Nodes, 2) + assert.Equal(t, "reliable", got.Nodes[0].Model) +} diff --git a/service/intelligent_routing/policy_control.go b/service/intelligent_routing/policy_control.go new file mode 100644 index 000000000000..db7f88487ef6 --- /dev/null +++ b/service/intelligent_routing/policy_control.go @@ -0,0 +1,251 @@ +package intelligent_routing + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" +) + +type PolicyRepository interface { + CreateDraft(policy model.IntelligentRoutingPolicy) (model.IntelligentRoutingPolicy, error) + UpdateDraft(id int64, updatedAt time.Time, config, checksum string) (model.IntelligentRoutingPolicy, error) + GetPolicy(id int64) (model.IntelligentRoutingPolicy, error) + GetPolicyByVersion(version int) (model.IntelligentRoutingPolicy, error) + Publish(id int64, administratorID int, note string) (model.IntelligentRoutingPolicy, error) + Rollback(version int, administratorID int, note string) (model.IntelligentRoutingPolicy, error) + GetRollout() (model.IntelligentRoutingRollout, error) + UpdateRollout(revision int64, rollout model.IntelligentRoutingRollout) (model.IntelligentRoutingRollout, error) +} + +type DatabasePolicyRepository struct{} + +func (DatabasePolicyRepository) CreateDraft(policy model.IntelligentRoutingPolicy) (model.IntelligentRoutingPolicy, error) { + return model.CreateIntelligentRoutingDraft(policy) +} + +func (DatabasePolicyRepository) UpdateDraft(id int64, updatedAt time.Time, config, checksum string) (model.IntelligentRoutingPolicy, error) { + return model.UpdateIntelligentRoutingDraft(id, updatedAt, config, checksum) +} + +func (DatabasePolicyRepository) GetPolicy(id int64) (model.IntelligentRoutingPolicy, error) { + return model.GetIntelligentRoutingPolicy(id) +} + +func (DatabasePolicyRepository) GetPolicyByVersion(version int) (model.IntelligentRoutingPolicy, error) { + return model.GetIntelligentRoutingPolicyByVersion(version) +} + +func (DatabasePolicyRepository) Publish(id int64, administratorID int, note string) (model.IntelligentRoutingPolicy, error) { + return model.PublishIntelligentRoutingPolicy(id, administratorID, note) +} + +func (DatabasePolicyRepository) Rollback(version int, administratorID int, note string) (model.IntelligentRoutingPolicy, error) { + return model.RollbackIntelligentRoutingPolicy(version, administratorID, note) +} + +func (DatabasePolicyRepository) GetRollout() (model.IntelligentRoutingRollout, error) { + return model.GetIntelligentRoutingRollout() +} + +func (DatabasePolicyRepository) UpdateRollout(revision int64, rollout model.IntelligentRoutingRollout) (model.IntelligentRoutingRollout, error) { + return model.UpdateIntelligentRoutingRollout(revision, rollout) +} + +type RuntimeRollout struct { + Exists bool + Revision int64 + PolicyVersion int + Enabled bool + Mode string + TrafficPercent int + UserGroups []string + TokenGroups []string +} + +type RuntimePolicySnapshot struct { + DeploymentSalt string + PolicyID int64 + Checksum string + Config routingsetting.Config + Rollout RuntimeRollout +} + +type PolicyControl struct { + repository PolicyRepository + salt string + snapshot atomic.Pointer[RuntimePolicySnapshot] +} + +var DefaultPolicyControl = NewPolicyControl( + DatabasePolicyRepository{}, + common.GetEnvOrDefaultString("INTELLIGENT_ROUTING_DEPLOYMENT_SALT", "intelligent-routing"), +) + +func NewPolicyControl(repository PolicyRepository, deploymentSalt string) *PolicyControl { + control := &PolicyControl{repository: repository, salt: deploymentSalt} + control.snapshot.Store(&RuntimePolicySnapshot{DeploymentSalt: deploymentSalt}) + return control +} + +func (control *PolicyControl) CreateDraft(_ context.Context, raw string, administratorID int) (model.IntelligentRoutingPolicy, []ValidationIssue, error) { + validated, issues := ValidatePolicyDocument(raw) + if len(issues) > 0 { + return model.IntelligentRoutingPolicy{}, issues, nil + } + policy, err := control.repository.CreateDraft(model.IntelligentRoutingPolicy{ + Config: validated.JSON, Checksum: validated.Checksum, CreatedBy: administratorID, + }) + return policy, nil, err +} + +func (control *PolicyControl) UpdateDraft(_ context.Context, id int64, updatedAt time.Time, raw string) (model.IntelligentRoutingPolicy, []ValidationIssue, error) { + validated, issues := ValidatePolicyDocument(raw) + if len(issues) > 0 { + return model.IntelligentRoutingPolicy{}, issues, nil + } + policy, err := control.repository.UpdateDraft(id, updatedAt, validated.JSON, validated.Checksum) + return policy, nil, err +} + +func (control *PolicyControl) Publish(ctx context.Context, id int64, administratorID int, changeNote string) (model.IntelligentRoutingPolicy, []ValidationIssue, error) { + if strings.TrimSpace(changeNote) == "" { + return model.IntelligentRoutingPolicy{}, []ValidationIssue{{Code: "change_note.required", Field: "change_note", Message: "Change note is required"}}, nil + } + policy, err := control.repository.GetPolicy(id) + if err != nil { + return model.IntelligentRoutingPolicy{}, nil, err + } + validated, issues := ValidatePolicyDocument(policy.Config) + if len(issues) > 0 { + return model.IntelligentRoutingPolicy{}, issues, nil + } + if validated.Checksum != policy.Checksum { + return model.IntelligentRoutingPolicy{}, []ValidationIssue{{Code: "policy.checksum_mismatch", Field: "policy", Message: "Policy checksum does not match its content"}}, nil + } + published, err := control.repository.Publish(id, administratorID, strings.TrimSpace(changeNote)) + if err != nil { + return model.IntelligentRoutingPolicy{}, nil, err + } + if refreshErr := control.RefreshSnapshot(ctx); refreshErr != nil && !errors.Is(refreshErr, model.ErrIntelligentRoutingRolloutNotFound) { + return published, nil, refreshErr + } + return published, nil, nil +} + +func (control *PolicyControl) Rollback(ctx context.Context, version, administratorID int, changeNote string) (model.IntelligentRoutingPolicy, []ValidationIssue, error) { + if strings.TrimSpace(changeNote) == "" { + return model.IntelligentRoutingPolicy{}, []ValidationIssue{{Code: "change_note.required", Field: "change_note", Message: "Change note is required"}}, nil + } + rolledBack, err := control.repository.Rollback(version, administratorID, strings.TrimSpace(changeNote)) + if err != nil { + return model.IntelligentRoutingPolicy{}, nil, err + } + if refreshErr := control.RefreshSnapshot(ctx); refreshErr != nil && !errors.Is(refreshErr, model.ErrIntelligentRoutingRolloutNotFound) { + return rolledBack, nil, refreshErr + } + return rolledBack, nil, nil +} + +func (control *PolicyControl) UpdateRollout(ctx context.Context, revision int64, rollout model.IntelligentRoutingRollout) (model.IntelligentRoutingRollout, []ValidationIssue, error) { + issues := validateRollout(rollout) + if len(issues) > 0 { + return model.IntelligentRoutingRollout{}, issues, nil + } + if rollout.PolicyVersion > 0 { + policy, err := control.repository.GetPolicyByVersion(rollout.PolicyVersion) + if err != nil { + return model.IntelligentRoutingRollout{}, nil, err + } + if policy.Status != model.IntelligentRoutingPolicyActive && rollout.Enabled { + return model.IntelligentRoutingRollout{}, []ValidationIssue{{Code: "policy_version.not_active", Field: "policy_version", Message: "Enabled rollout requires the active policy"}}, nil + } + } + updated, err := control.repository.UpdateRollout(revision, rollout) + if err != nil { + return model.IntelligentRoutingRollout{}, nil, err + } + if err := control.RefreshSnapshot(ctx); err != nil { + return updated, nil, err + } + return updated, nil, nil +} + +func validateRollout(rollout model.IntelligentRoutingRollout) []ValidationIssue { + if rollout.Mode != model.IntelligentRoutingModeShadow && rollout.Mode != model.IntelligentRoutingModeLive { + return []ValidationIssue{{Code: "mode.invalid", Field: "mode", Message: "Rollout mode must be shadow or live"}} + } + if rollout.TrafficPercent < 0 || rollout.TrafficPercent > 100 { + return []ValidationIssue{{Code: "traffic_percent.out_of_range", Field: "traffic_percent", Message: "Traffic percentage must be between 0 and 100"}} + } + if rollout.Enabled && rollout.PolicyVersion < 1 { + return []ValidationIssue{{Code: "policy_version.required", Field: "policy_version", Message: "Enabled rollout requires a policy version"}} + } + return nil +} + +func (control *PolicyControl) RefreshSnapshot(_ context.Context) error { + rollout, err := control.repository.GetRollout() + if errors.Is(err, model.ErrIntelligentRoutingRolloutNotFound) { + control.snapshot.Store(&RuntimePolicySnapshot{DeploymentSalt: control.salt}) + return nil + } + if err != nil { + return err + } + snapshot := RuntimePolicySnapshot{DeploymentSalt: control.salt, Rollout: RuntimeRollout{ + Exists: true, Revision: rollout.Revision, PolicyVersion: rollout.PolicyVersion, Enabled: rollout.Enabled, + Mode: rollout.Mode, TrafficPercent: rollout.TrafficPercent, + }} + if err := common.UnmarshalJsonStr(rollout.UserGroups, &snapshot.Rollout.UserGroups); err != nil && strings.TrimSpace(rollout.UserGroups) != "" { + return err + } + if err := common.UnmarshalJsonStr(rollout.TokenGroups, &snapshot.Rollout.TokenGroups); err != nil && strings.TrimSpace(rollout.TokenGroups) != "" { + return err + } + if rollout.PolicyVersion > 0 { + policy, err := control.repository.GetPolicyByVersion(rollout.PolicyVersion) + if err != nil { + return err + } + validated, issues := ValidatePolicyDocument(policy.Config) + if len(issues) > 0 || validated.Checksum != policy.Checksum { + return errors.New("stored intelligent routing policy failed validation") + } + snapshot.PolicyID = policy.Id + snapshot.Checksum = policy.Checksum + snapshot.Config = validated.Config + } + control.snapshot.Store(&snapshot) + return nil +} + +func (control *PolicyControl) Snapshot() RuntimePolicySnapshot { + current := control.snapshot.Load() + if current == nil { + return RuntimePolicySnapshot{DeploymentSalt: control.salt} + } + copy := *current + copy.Rollout.UserGroups = append([]string(nil), current.Rollout.UserGroups...) + copy.Rollout.TokenGroups = append([]string(nil), current.Rollout.TokenGroups...) + copy.Config = cloneRoutingConfig(current.Config) + return copy +} + +func cloneRoutingConfig(input routingsetting.Config) routingsetting.Config { + input.Models = append([]routingsetting.ModelPolicy(nil), input.Models...) + for index := range input.Models { + input.Models[index].Capabilities = append([]string(nil), input.Models[index].Capabilities...) + } + thresholds := make(map[routingsetting.TaskType]float64, len(input.QualityThresholds)) + for task, value := range input.QualityThresholds { + thresholds[task] = value + } + input.QualityThresholds = thresholds + return input +} diff --git a/service/intelligent_routing/policy_control_test.go b/service/intelligent_routing/policy_control_test.go new file mode 100644 index 000000000000..c549c1eb89a4 --- /dev/null +++ b/service/intelligent_routing/policy_control_test.go @@ -0,0 +1,126 @@ +package intelligent_routing + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/QuantumNous/new-api/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type policyRepositoryFixture struct { + policies map[int64]model.IntelligentRoutingPolicy + rollout model.IntelligentRoutingRollout + createCalls int + err error + nextID int64 +} + +func (repo *policyRepositoryFixture) CreateDraft(policy model.IntelligentRoutingPolicy) (model.IntelligentRoutingPolicy, error) { + repo.createCalls++ + repo.nextID++ + policy.Id = repo.nextID + policy.Status = model.IntelligentRoutingPolicyDraft + policy.UpdatedAt = time.Now() + if repo.policies == nil { + repo.policies = make(map[int64]model.IntelligentRoutingPolicy) + } + repo.policies[policy.Id] = policy + return policy, repo.err +} + +func (repo *policyRepositoryFixture) UpdateDraft(id int64, updatedAt time.Time, config, checksum string) (model.IntelligentRoutingPolicy, error) { + policy := repo.policies[id] + policy.Config, policy.Checksum = config, checksum + repo.policies[id] = policy + return policy, repo.err +} + +func (repo *policyRepositoryFixture) GetPolicy(id int64) (model.IntelligentRoutingPolicy, error) { + policy, ok := repo.policies[id] + if !ok { + return policy, model.ErrIntelligentRoutingPolicyNotFound + } + return policy, repo.err +} + +func (repo *policyRepositoryFixture) GetPolicyByVersion(version int) (model.IntelligentRoutingPolicy, error) { + for _, policy := range repo.policies { + if policy.Version == version { + return policy, repo.err + } + } + return model.IntelligentRoutingPolicy{}, model.ErrIntelligentRoutingPolicyNotFound +} + +func (repo *policyRepositoryFixture) Publish(id int64, administratorID int, note string) (model.IntelligentRoutingPolicy, error) { + policy := repo.policies[id] + policy.Version, policy.Status, policy.ChangeNote = 1, model.IntelligentRoutingPolicyActive, note + repo.policies[id] = policy + return policy, repo.err +} + +func (repo *policyRepositoryFixture) Rollback(version int, administratorID int, note string) (model.IntelligentRoutingPolicy, error) { + policy, err := repo.GetPolicyByVersion(version) + if err != nil { + return policy, err + } + policy.Version, policy.SourceVersion, policy.ChangeNote = version+1, version, note + repo.nextID++ + policy.Id = repo.nextID + repo.policies[policy.Id] = policy + return policy, repo.err +} + +func (repo *policyRepositoryFixture) GetRollout() (model.IntelligentRoutingRollout, error) { + if repo.err != nil { + return model.IntelligentRoutingRollout{}, repo.err + } + if repo.rollout.Id == 0 { + return model.IntelligentRoutingRollout{}, model.ErrIntelligentRoutingRolloutNotFound + } + return repo.rollout, nil +} + +func (repo *policyRepositoryFixture) UpdateRollout(revision int64, rollout model.IntelligentRoutingRollout) (model.IntelligentRoutingRollout, error) { + if repo.err != nil { + return model.IntelligentRoutingRollout{}, repo.err + } + rollout.Id, rollout.Revision = 1, revision+1 + repo.rollout = rollout + return rollout, nil +} + +func TestPolicyControlRejectsInvalidDraftBeforeRepositoryWrite(t *testing.T) { + repo := &policyRepositoryFixture{} + control := NewPolicyControl(repo, "deployment-salt") + + _, issues, err := control.CreateDraft(context.Background(), `{"max_attempts":99}`, 7) + require.NoError(t, err) + require.NotEmpty(t, issues) + assert.Zero(t, repo.createCalls) +} + +func TestPolicyControlRefreshRetainsLastValidSnapshot(t *testing.T) { + repo := &policyRepositoryFixture{} + control := NewPolicyControl(repo, "deployment-salt") + draft, issues, err := control.CreateDraft(context.Background(), `{"models":[{"model":"cheap","tier":1,"context_limit":4096}]}`, 7) + require.NoError(t, err) + require.Empty(t, issues) + policy := repo.policies[draft.Id] + policy.Version, policy.Status = 1, model.IntelligentRoutingPolicyActive + repo.policies[draft.Id] = policy + repo.rollout = model.IntelligentRoutingRollout{Id: 1, Revision: 2, PolicyVersion: 1, Enabled: true, Mode: model.IntelligentRoutingModeShadow, TrafficPercent: 100, UserGroups: `[]`, TokenGroups: `[]`} + + require.NoError(t, control.RefreshSnapshot(context.Background())) + before := control.Snapshot() + assert.True(t, before.Rollout.Enabled) + + repo.err = errors.New("database unavailable") + assert.Error(t, control.RefreshSnapshot(context.Background())) + after := control.Snapshot() + assert.Equal(t, before.Rollout.Revision, after.Rollout.Revision) +} diff --git a/service/intelligent_routing/policy_document.go b/service/intelligent_routing/policy_document.go new file mode 100644 index 000000000000..d4d65354105d --- /dev/null +++ b/service/intelligent_routing/policy_document.go @@ -0,0 +1,78 @@ +package intelligent_routing + +import ( + "crypto/sha256" + "fmt" + "sort" + + "github.com/QuantumNous/new-api/common" + routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" +) + +type ValidationIssue struct { + Code string `json:"code"` + Field string `json:"field"` + Message string `json:"message"` +} + +type ValidatedPolicy struct { + Config routingsetting.Config + JSON string + Checksum string +} + +func ValidatePolicyDocument(raw string) (ValidatedPolicy, []ValidationIssue) { + if len(raw) > routingsetting.MaxPolicyDocumentBytes { + return ValidatedPolicy{}, []ValidationIssue{{Code: "policy.too_large", Field: "policy", Message: "Policy document is too large"}} + } + var input routingsetting.Config + if err := common.UnmarshalJsonStr(raw, &input); err != nil { + return ValidatedPolicy{}, []ValidationIssue{{Code: "policy.invalid_json", Field: "policy", Message: "Policy document is not valid JSON"}} + } + if input.MaxAttempts > routingsetting.MaxAttempts || input.MaxAttempts < 0 { + return ValidatedPolicy{}, []ValidationIssue{{Code: "max_attempts.out_of_range", Field: "max_attempts", Message: "Maximum attempts is out of range"}} + } + if input.MaxEndpointsPerModel > routingsetting.MaxEndpointsPerModel || input.MaxEndpointsPerModel < 0 { + return ValidatedPolicy{}, []ValidationIssue{{Code: "max_endpoints_per_model.out_of_range", Field: "max_endpoints_per_model", Message: "Maximum endpoints per model is out of range"}} + } + allowedCapabilities := map[string]struct{}{ + string(CapabilityTools): {}, string(CapabilityJSONSchema): {}, string(CapabilityVision): {}, string(CapabilityAudio): {}, + } + for modelIndex, policy := range input.Models { + for capabilityIndex, capability := range policy.Capabilities { + if _, ok := allowedCapabilities[capability]; !ok { + return ValidatedPolicy{}, []ValidationIssue{{ + Code: "models.capability.unknown", Field: fmt.Sprintf("models[%d].capabilities[%d]", modelIndex, capabilityIndex), + Message: "Model capability is unknown", + }} + } + } + } + + normalized, err := routingsetting.Normalize(input) + if err != nil { + return ValidatedPolicy{}, []ValidationIssue{{Code: "policy.invalid", Field: "policy", Message: "Policy document contains invalid values"}} + } + canonical, err := CanonicalPolicyJSON(normalized) + if err != nil { + return ValidatedPolicy{}, []ValidationIssue{{Code: "policy.canonicalization_failed", Field: "policy", Message: "Policy document could not be canonicalized"}} + } + sum := sha256.Sum256([]byte(canonical)) + return ValidatedPolicy{Config: normalized, JSON: canonical, Checksum: fmt.Sprintf("%x", sum)}, nil +} + +func CanonicalPolicyJSON(config routingsetting.Config) (string, error) { + config.Models = append([]routingsetting.ModelPolicy(nil), config.Models...) + for index := range config.Models { + config.Models[index].Capabilities = append([]string(nil), config.Models[index].Capabilities...) + sort.Strings(config.Models[index].Capabilities) + } + sort.Slice(config.Models, func(i, j int) bool { + return config.Models[i].Model < config.Models[j].Model + }) + data, err := common.Marshal(config) + if err != nil { + return "", err + } + return string(data), nil +} diff --git a/service/intelligent_routing/policy_document_test.go b/service/intelligent_routing/policy_document_test.go new file mode 100644 index 000000000000..927c11c05003 --- /dev/null +++ b/service/intelligent_routing/policy_document_test.go @@ -0,0 +1,43 @@ +package intelligent_routing + +import ( + "strings" + "testing" + + routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidatePolicyDocumentReturnsStructuredIssues(t *testing.T) { + tests := []struct { + name string + raw string + code string + field string + }{ + {name: "malformed", raw: `{`, code: "policy.invalid_json", field: "policy"}, + {name: "too large", raw: `{"padding":"` + strings.Repeat("x", routingsetting.MaxPolicyDocumentBytes) + `"}`, code: "policy.too_large", field: "policy"}, + {name: "attempts", raw: `{"max_attempts":99}`, code: "max_attempts.out_of_range", field: "max_attempts"}, + {name: "capability", raw: `{"models":[{"model":"cheap","tier":1,"context_limit":4096,"capabilities":["telepathy"]}]}`, code: "models.capability.unknown", field: "models[0].capabilities[0]"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, issues := ValidatePolicyDocument(test.raw) + require.NotEmpty(t, issues) + assert.Equal(t, test.code, issues[0].Code) + assert.Equal(t, test.field, issues[0].Field) + }) + } +} + +func TestCanonicalPolicyJSONProducesStableChecksum(t *testing.T) { + first, issues := ValidatePolicyDocument(`{"models":[{"model":"b","tier":1,"context_limit":4096,"capabilities":["tools","json_schema"]},{"model":"a","tier":0,"context_limit":2048}]}`) + require.Empty(t, issues) + second, issues := ValidatePolicyDocument(`{"models":[{"model":"a","tier":0,"context_limit":2048},{"model":"b","tier":1,"context_limit":4096,"capabilities":["json_schema","tools"]}]}`) + require.Empty(t, issues) + + assert.Equal(t, first.Checksum, second.Checksum) + assert.Equal(t, first.JSON, second.JSON) +} diff --git a/service/intelligent_routing/policy_refresh.go b/service/intelligent_routing/policy_refresh.go new file mode 100644 index 000000000000..2f2847eeed12 --- /dev/null +++ b/service/intelligent_routing/policy_refresh.go @@ -0,0 +1,38 @@ +package intelligent_routing + +import ( + "context" + "time" + + "github.com/QuantumNous/new-api/common" +) + +func StartPolicyRefresh(ctx context.Context, control *PolicyControl, interval time.Duration) { + if interval <= 0 { + interval = time.Minute + } + ticker := time.NewTicker(interval) + go func() { + defer ticker.Stop() + runPolicyRefresh(ctx, control, ticker.C) + }() +} + +func runPolicyRefresh(ctx context.Context, control *PolicyControl, triggers <-chan time.Time) { + failed := false + for { + select { + case <-ctx.Done(): + return + case <-triggers: + if err := control.RefreshSnapshot(ctx); err != nil { + if !failed { + common.SysError("failed to refresh intelligent routing policy snapshot: " + err.Error()) + } + failed = true + continue + } + failed = false + } + } +} diff --git a/service/intelligent_routing/policy_refresh_test.go b/service/intelligent_routing/policy_refresh_test.go new file mode 100644 index 000000000000..5a747f19d1d1 --- /dev/null +++ b/service/intelligent_routing/policy_refresh_test.go @@ -0,0 +1,30 @@ +package intelligent_routing + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestPolicyRefreshStopsAfterCancellation(t *testing.T) { + repo := &policyRepositoryFixture{} + control := NewPolicyControl(repo, "salt") + triggers := make(chan time.Time, 2) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + runPolicyRefresh(ctx, control, triggers) + close(done) + }() + + triggers <- time.Now() + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("refresh loop did not stop after cancellation") + } + assert.False(t, control.Snapshot().Rollout.Enabled) +} diff --git a/service/intelligent_routing/quality.go b/service/intelligent_routing/quality.go new file mode 100644 index 000000000000..105063435433 --- /dev/null +++ b/service/intelligent_routing/quality.go @@ -0,0 +1,48 @@ +package intelligent_routing + +import "sync" + +type qualityKey struct { + model string + task TaskType +} + +type qualityCounts struct { + successes int + samples int +} + +type QualityTracker struct { + mu sync.RWMutex + counts map[qualityKey]qualityCounts +} + +var DefaultQualityTracker QualityTracker + +func (tracker *QualityTracker) Record(model string, task TaskType, success bool) { + if model == "" || task == "" { + return + } + tracker.mu.Lock() + defer tracker.mu.Unlock() + if tracker.counts == nil { + tracker.counts = make(map[qualityKey]qualityCounts) + } + key := qualityKey{model: model, task: task} + counts := tracker.counts[key] + counts.samples++ + if success { + counts.successes++ + } + tracker.counts[key] = counts +} + +func (tracker *QualityTracker) Predict(model string, task TaskType, prior float64) float64 { + tracker.mu.RLock() + counts := tracker.counts[qualityKey{model: model, task: task}] + tracker.mu.RUnlock() + if counts.samples < 30 { + return prior + } + return float64(counts.successes+8) / float64(counts.samples+10) +} diff --git a/service/intelligent_routing/quality_test.go b/service/intelligent_routing/quality_test.go new file mode 100644 index 000000000000..020bdd77a2da --- /dev/null +++ b/service/intelligent_routing/quality_test.go @@ -0,0 +1,26 @@ +package intelligent_routing + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestQualityTrackerUsesPriorUntilThirtySamplesThenBetaSmoothing(t *testing.T) { + var tracker QualityTracker + for i := 0; i < 29; i++ { + tracker.Record("cheap", TaskSummary, i < 20) + } + assert.InDelta(t, .92, tracker.Predict("cheap", TaskSummary, .92), .0001) + tracker.Record("cheap", TaskSummary, true) + assert.InDelta(t, float64(21+8)/float64(30+10), tracker.Predict("cheap", TaskSummary, .92), .0001) +} + +func TestQualityTrackerSeparatesModelsAndTasks(t *testing.T) { + var tracker QualityTracker + for i := 0; i < 30; i++ { + tracker.Record("cheap", TaskCode, false) + } + assert.InDelta(t, .8, tracker.Predict("cheap", TaskSummary, .8), .0001) + assert.InDelta(t, .2, tracker.Predict("cheap", TaskCode, .8), .0001) +} diff --git a/service/intelligent_routing/rollout.go b/service/intelligent_routing/rollout.go new file mode 100644 index 000000000000..34fe0c6f47dc --- /dev/null +++ b/service/intelligent_routing/rollout.go @@ -0,0 +1,54 @@ +package intelligent_routing + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/binary" + "fmt" +) + +type RolloutSubject struct { + AccountID int + TokenID int + UserGroup string + TokenGroup string +} + +type RolloutDecision struct { + Selected bool + Bucket int + Mode string + PolicyVersion int + Revision int64 +} + +func ResolveRollout(snapshot RuntimePolicySnapshot, subject RolloutSubject) RolloutDecision { + decision := RolloutDecision{ + Mode: snapshot.Rollout.Mode, PolicyVersion: snapshot.Rollout.PolicyVersion, Revision: snapshot.Rollout.Revision, + } + if !snapshot.Rollout.Exists || !snapshot.Rollout.Enabled || snapshot.Rollout.TrafficPercent <= 0 { + return decision + } + if !rolloutGroupMatches(snapshot.Rollout.UserGroups, subject.UserGroup) || + !rolloutGroupMatches(snapshot.Rollout.TokenGroups, subject.TokenGroup) { + return decision + } + mac := hmac.New(sha256.New, []byte(snapshot.DeploymentSalt)) + _, _ = fmt.Fprintf(mac, "%d/%d/%d", snapshot.Rollout.PolicyVersion, subject.AccountID, subject.TokenID) + digest := mac.Sum(nil) + decision.Bucket = int(binary.BigEndian.Uint64(digest[:8]) % 100) + decision.Selected = decision.Bucket < snapshot.Rollout.TrafficPercent + return decision +} + +func rolloutGroupMatches(allowed []string, actual string) bool { + if len(allowed) == 0 { + return true + } + for _, candidate := range allowed { + if candidate == actual { + return true + } + } + return false +} diff --git a/service/intelligent_routing/rollout_test.go b/service/intelligent_routing/rollout_test.go new file mode 100644 index 000000000000..fea1c271c0b2 --- /dev/null +++ b/service/intelligent_routing/rollout_test.go @@ -0,0 +1,39 @@ +package intelligent_routing + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResolveRolloutUsesStableSubjectBucket(t *testing.T) { + snapshot := RuntimePolicySnapshot{DeploymentSalt: "salt", Rollout: RuntimeRollout{ + Exists: true, Revision: 3, PolicyVersion: 2, Enabled: true, Mode: "live", TrafficPercent: 100, + UserGroups: []string{"default"}, TokenGroups: []string{"auto"}, + }} + subject := RolloutSubject{AccountID: 42, TokenID: 9, UserGroup: "default", TokenGroup: "auto"} + + first := ResolveRollout(snapshot, subject) + second := ResolveRollout(snapshot, subject) + + assert.True(t, first.Selected) + assert.Equal(t, first.Bucket, second.Bucket) + assert.Equal(t, "live", first.Mode) +} + +func TestResolveRolloutRejectsDisabledExcludedAndZeroPercent(t *testing.T) { + base := RuntimePolicySnapshot{DeploymentSalt: "salt", Rollout: RuntimeRollout{ + Exists: true, PolicyVersion: 1, Enabled: true, Mode: "shadow", TrafficPercent: 100, + UserGroups: []string{"allowed"}, TokenGroups: []string{"auto"}, + }} + subject := RolloutSubject{AccountID: 1, TokenID: 2, UserGroup: "other", TokenGroup: "auto"} + assert.False(t, ResolveRollout(base, subject).Selected) + + base.Rollout.UserGroups = nil + base.Rollout.Enabled = false + assert.False(t, ResolveRollout(base, subject).Selected) + + base.Rollout.Enabled = true + base.Rollout.TrafficPercent = 0 + assert.False(t, ResolveRollout(base, subject).Selected) +} diff --git a/service/intelligent_routing/runtime_store.go b/service/intelligent_routing/runtime_store.go new file mode 100644 index 000000000000..e75b76c97636 --- /dev/null +++ b/service/intelligent_routing/runtime_store.go @@ -0,0 +1,41 @@ +package intelligent_routing + +import "sync/atomic" + +type HealthStore interface { + Record(channelID int, success bool) + Snapshot(channelID int) HealthSnapshot +} + +type QualityStore interface { + Record(model string, task TaskType, success bool) + Predict(model string, task TaskType, prior float64) float64 +} + +type StickyStore interface { + Record(key string, task TaskType, route StickyRoute) + Get(key string, task TaskType) (StickyRoute, bool) + RecordValidationFailure(key string) +} + +type SharedRuntime struct { + configured atomic.Bool + healthy atomic.Bool +} + +var DefaultSharedRuntime SharedRuntime + +func (runtime *SharedRuntime) Configure(configured bool) { + runtime.configured.Store(configured) + if !configured { + runtime.healthy.Store(false) + } +} + +func (runtime *SharedRuntime) SetHealthy(healthy bool) { + runtime.healthy.Store(healthy) +} + +func (runtime *SharedRuntime) Ready() bool { + return runtime.configured.Load() && runtime.healthy.Load() +} diff --git a/service/intelligent_routing/runtime_store_test.go b/service/intelligent_routing/runtime_store_test.go new file mode 100644 index 000000000000..e26625fb80b8 --- /dev/null +++ b/service/intelligent_routing/runtime_store_test.go @@ -0,0 +1,28 @@ +package intelligent_routing + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSharedRuntimeRequiresConfiguredHealthyState(t *testing.T) { + runtime := &SharedRuntime{} + assert.False(t, runtime.Ready()) + + runtime.Configure(true) + assert.False(t, runtime.Ready()) + + runtime.SetHealthy(true) + assert.True(t, runtime.Ready()) + + runtime.SetHealthy(false) + assert.False(t, runtime.Ready()) +} + +func TestSharedRuntimeDisabledNeverBecomesReady(t *testing.T) { + runtime := &SharedRuntime{} + runtime.Configure(false) + runtime.SetHealthy(true) + assert.False(t, runtime.Ready()) +} diff --git a/service/intelligent_routing/stickiness.go b/service/intelligent_routing/stickiness.go new file mode 100644 index 000000000000..f7ad648b042a --- /dev/null +++ b/service/intelligent_routing/stickiness.go @@ -0,0 +1,94 @@ +package intelligent_routing + +import ( + "crypto/sha256" + "fmt" + "sync" + "time" +) + +const stickinessTTL = 30 * time.Minute + +type StickyRoute struct { + Model string + ChannelID int +} + +type stickyEntry struct { + route StickyRoute + task TaskType + expiresAt time.Time + validationFailures int +} + +func (store *StickinessStore) RecordValidationFailure(key string) { + if key == "" { + return + } + store.mu.Lock() + defer store.mu.Unlock() + entry, ok := store.entries[key] + if !ok { + return + } + entry.validationFailures++ + if entry.validationFailures >= 2 { + delete(store.entries, key) + return + } + store.entries[key] = entry +} + +type StickinessStore struct { + mu sync.Mutex + entries map[string]stickyEntry +} + +var DefaultStickinessStore StickinessStore + +func (store *StickinessStore) RecordAt(key string, task TaskType, route StickyRoute, now time.Time) { + if key == "" || route.Model == "" || route.ChannelID == 0 { + return + } + store.mu.Lock() + defer store.mu.Unlock() + if store.entries == nil { + store.entries = make(map[string]stickyEntry) + } + store.entries[key] = stickyEntry{route: route, task: task, expiresAt: now.Add(stickinessTTL)} +} + +func (store *StickinessStore) Record(key string, task TaskType, route StickyRoute) { + store.RecordAt(key, task, route, time.Now()) +} + +func (store *StickinessStore) GetAt(key string, task TaskType, now time.Time) (StickyRoute, bool) { + if key == "" { + return StickyRoute{}, false + } + store.mu.Lock() + defer store.mu.Unlock() + entry, ok := store.entries[key] + if !ok || entry.task != task || !now.Before(entry.expiresAt) { + if ok && !now.Before(entry.expiresAt) { + delete(store.entries, key) + } + return StickyRoute{}, false + } + return entry.route, true +} + +func (store *StickinessStore) Get(key string, task TaskType) (StickyRoute, bool) { + return store.GetAt(key, task, time.Now()) +} + +func ConversationKey(account, explicitSession, firstMessage string) string { + seed := firstMessage + if explicitSession != "" { + seed = explicitSession + } + if account == "" || seed == "" { + return "" + } + return fmt.Sprintf("%x", sha256.Sum256([]byte(account+"\x00"+seed))) +} diff --git a/service/intelligent_routing/stickiness_test.go b/service/intelligent_routing/stickiness_test.go new file mode 100644 index 000000000000..8daaa4faa439 --- /dev/null +++ b/service/intelligent_routing/stickiness_test.go @@ -0,0 +1,42 @@ +package intelligent_routing + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStickinessStoreReturnsHealthyRouteForSameTaskUntilExpiry(t *testing.T) { + var store StickinessStore + now := time.Unix(1000, 0) + store.RecordAt("session", TaskSummary, StickyRoute{Model: "cached", ChannelID: 7}, now) + route, ok := store.GetAt("session", TaskSummary, now.Add(10*time.Minute)) + require.True(t, ok) + assert.Equal(t, "cached", route.Model) + _, changedTask := store.GetAt("session", TaskCode, now.Add(time.Minute)) + assert.False(t, changedTask) + _, expired := store.GetAt("session", TaskSummary, now.Add(31*time.Minute)) + assert.False(t, expired) +} + +func TestConversationKeyPrefersExplicitSessionAndOtherwiseStaysDeterministic(t *testing.T) { + explicit := ConversationKey("account", "explicit-session", "ignored") + assert.Equal(t, ConversationKey("account", "explicit-session", "different"), explicit) + derived := ConversationKey("account", "", "first user message") + assert.NotEmpty(t, derived) + assert.Equal(t, derived, ConversationKey("account", "", "first user message")) + assert.NotEqual(t, derived, ConversationKey("other-account", "", "first user message")) +} + +func TestStickinessStoreInvalidatesAfterTwoConsecutiveValidationFailures(t *testing.T) { + var store StickinessStore + store.Record("session", TaskGeneral, StickyRoute{Model: "cached", ChannelID: 7}) + store.RecordValidationFailure("session") + _, ok := store.Get("session", TaskGeneral) + require.True(t, ok) + store.RecordValidationFailure("session") + _, ok = store.Get("session", TaskGeneral) + assert.False(t, ok) +} diff --git a/service/intelligent_routing/validation.go b/service/intelligent_routing/validation.go new file mode 100644 index 000000000000..349bef0a5559 --- /dev/null +++ b/service/intelligent_routing/validation.go @@ -0,0 +1,242 @@ +package intelligent_routing + +import ( + "errors" + "fmt" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/types" +) + +func ValidateResponse(request dto.Request, format types.RelayFormat, body []byte) error { + if len(strings.TrimSpace(string(body))) == 0 { + return errors.New("empty intelligent routing response") + } + switch format { + case types.RelayFormatOpenAI: + return validateChatResponse(request, body) + case types.RelayFormatOpenAIResponses, types.RelayFormatOpenAIResponsesCompaction: + return validateResponsesResponse(request, body) + default: + return nil + } +} + +func validateChatResponse(request dto.Request, body []byte) error { + var response struct { + Choices []struct { + FinishReason string `json:"finish_reason"` + Message struct { + Content any `json:"content"` + ToolCalls []struct { + Function struct { + Name string `json:"name"` + Arguments string `json:"arguments"` + } `json:"function"` + } `json:"tool_calls"` + } `json:"message"` + } `json:"choices"` + } + if err := common.Unmarshal(body, &response); err != nil { + return fmt.Errorf("invalid chat response JSON: %w", err) + } + if len(response.Choices) == 0 { + return errors.New("chat response has no choices") + } + choice := response.Choices[0] + if choice.FinishReason == "length" { + return errors.New("chat response was truncated") + } + content, _ := choice.Message.Content.(string) + if strings.TrimSpace(content) == "" && len(choice.Message.ToolCalls) == 0 { + return errors.New("chat response has no content or tool call") + } + if ExtractFeatures(Input{Request: request}).Task == TaskCode && strings.Count(content, "```")%2 != 0 { + return errors.New("code response has an incomplete code fence") + } + openAIRequest, _ := request.(*dto.GeneralOpenAIRequest) + if openAIRequest != nil && openAIRequest.ResponseFormat != nil && openAIRequest.ResponseFormat.Type == "json_schema" { + var structured any + if err := common.Unmarshal([]byte(content), &structured); err != nil { + return fmt.Errorf("structured response is not valid JSON: %w", err) + } + if len(openAIRequest.ResponseFormat.JsonSchema) > 0 { + var format dto.FormatJsonSchema + if err := common.Unmarshal(openAIRequest.ResponseFormat.JsonSchema, &format); err != nil { + return fmt.Errorf("request JSON schema is invalid: %w", err) + } + if err := validateJSONSchemaValue(structured, format.Schema, "$", true); err != nil { + return err + } + } + } + if openAIRequest == nil || len(choice.Message.ToolCalls) == 0 { + return nil + } + allowed := make(map[string]any, len(openAIRequest.Tools)) + for _, tool := range openAIRequest.Tools { + allowed[tool.Function.Name] = tool.Function.Parameters + } + for _, call := range choice.Message.ToolCalls { + parameterSchema, ok := allowed[call.Function.Name] + if !ok { + return fmt.Errorf("response called undeclared tool %q", call.Function.Name) + } + var arguments any + if err := common.Unmarshal([]byte(call.Function.Arguments), &arguments); err != nil { + return fmt.Errorf("tool %q arguments are not valid JSON: %w", call.Function.Name, err) + } + if parameterSchema != nil { + if err := validateJSONSchemaValue(arguments, parameterSchema, "$arguments", true); err != nil { + return fmt.Errorf("tool %q arguments failed schema validation: %w", call.Function.Name, err) + } + } + } + return nil +} + +func validateJSONSchemaValue(value, rawSchema any, path string, root bool) error { + schema, ok := rawSchema.(map[string]any) + if !ok { + return nil + } + typeName, _ := schema["type"].(string) + switch typeName { + case "object": + object, ok := value.(map[string]any) + if !ok { + return fmt.Errorf("JSON schema %s requires object", path) + } + if required, ok := schema["required"].([]any); ok { + for _, item := range required { + name, _ := item.(string) + if _, exists := object[name]; name != "" && !exists { + return fmt.Errorf("JSON schema %s is missing required field %q", path, name) + } + } + } + properties, _ := schema["properties"].(map[string]any) + for name, propertySchema := range properties { + if property, exists := object[name]; exists { + if err := validateJSONSchemaValue(property, propertySchema, path+"."+name, false); err != nil { + return err + } + } + } + case "array": + items, ok := value.([]any) + if !ok { + return fmt.Errorf("JSON schema %s requires array", path) + } + for i, item := range items { + if err := validateJSONSchemaValue(item, schema["items"], fmt.Sprintf("%s[%d]", path, i), false); err != nil { + return err + } + } + case "string": + if _, ok := value.(string); !ok { + return fmt.Errorf("JSON schema %s requires string", path) + } + case "number": + if _, ok := value.(float64); !ok { + return fmt.Errorf("JSON schema %s requires number", path) + } + case "integer": + number, ok := value.(float64) + if !ok || number != float64(int64(number)) { + return fmt.Errorf("JSON schema %s requires integer", path) + } + case "boolean": + if _, ok := value.(bool); !ok { + return fmt.Errorf("JSON schema %s requires boolean", path) + } + default: + if root && typeName == "" { + return errors.New("JSON schema has no root type") + } + } + return nil +} + +func validateResponsesResponse(request dto.Request, body []byte) error { + var response struct { + Status string `json:"status"` + Output []struct { + Type string `json:"type"` + Name string `json:"name"` + Arguments string `json:"arguments"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } `json:"output"` + } + if err := common.Unmarshal(body, &response); err != nil { + return fmt.Errorf("invalid responses API JSON: %w", err) + } + if response.Status == "incomplete" || response.Status == "failed" { + return fmt.Errorf("responses API returned %s status", response.Status) + } + if len(response.Output) == 0 { + return errors.New("responses API response has no output") + } + allowedTools := make(map[string]any) + var responseSchema any + if responsesRequest, ok := request.(*dto.OpenAIResponsesRequest); ok && len(responsesRequest.Tools) > 0 { + var tools []struct { + Name string `json:"name"` + Parameters any `json:"parameters"` + } + if err := common.Unmarshal(responsesRequest.Tools, &tools); err == nil { + for _, tool := range tools { + allowedTools[tool.Name] = tool.Parameters + } + } + } + if responsesRequest, ok := request.(*dto.OpenAIResponsesRequest); ok && len(responsesRequest.Text) > 0 { + var textConfig struct { + Format struct { + Type string `json:"type"` + Schema any `json:"schema"` + } `json:"format"` + } + if err := common.Unmarshal(responsesRequest.Text, &textConfig); err == nil && textConfig.Format.Type == "json_schema" { + responseSchema = textConfig.Format.Schema + } + } + for _, output := range response.Output { + if output.Type == "function_call" && output.Name != "" && output.Arguments != "" { + parameterSchema, declared := allowedTools[output.Name] + if len(allowedTools) > 0 && !declared { + return fmt.Errorf("response called undeclared tool %q", output.Name) + } + var arguments any + if err := common.Unmarshal([]byte(output.Arguments), &arguments); err != nil { + return fmt.Errorf("tool %q arguments are not valid JSON: %w", output.Name, err) + } + if parameterSchema != nil { + if err := validateJSONSchemaValue(arguments, parameterSchema, "$arguments", true); err != nil { + return fmt.Errorf("tool %q arguments failed schema validation: %w", output.Name, err) + } + } + return nil + } + for _, content := range output.Content { + if content.Type == "output_text" && strings.TrimSpace(content.Text) != "" { + if responseSchema != nil { + var structured any + if err := common.Unmarshal([]byte(content.Text), &structured); err != nil { + return fmt.Errorf("structured response is not valid JSON: %w", err) + } + if err := validateJSONSchemaValue(structured, responseSchema, "$", true); err != nil { + return err + } + } + return nil + } + } + } + return errors.New("responses API response has no usable output") +} diff --git a/service/intelligent_routing/validation_test.go b/service/intelligent_routing/validation_test.go new file mode 100644 index 000000000000..eed75b1b4fb2 --- /dev/null +++ b/service/intelligent_routing/validation_test.go @@ -0,0 +1,74 @@ +package intelligent_routing + +import ( + "testing" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateResponseRejectsEmptyTruncatedAndMalformedStructuredOutput(t *testing.T) { + plain := &dto.GeneralOpenAIRequest{} + assert.Error(t, ValidateResponse(plain, types.RelayFormatOpenAI, []byte(`{"choices":[]}`))) + assert.Error(t, ValidateResponse(plain, types.RelayFormatOpenAI, []byte(`{"choices":[{"finish_reason":"length","message":{"content":"partial"}}]}`))) + + structured := &dto.GeneralOpenAIRequest{ResponseFormat: &dto.ResponseFormat{Type: "json_schema"}} + assert.Error(t, ValidateResponse(structured, types.RelayFormatOpenAI, []byte(`{"choices":[{"finish_reason":"stop","message":{"content":"not-json"}}]}`))) + require.NoError(t, ValidateResponse(structured, types.RelayFormatOpenAI, []byte(`{"choices":[{"finish_reason":"stop","message":{"content":"{\"ok\":true}"}}]}`))) +} + +func TestValidateResponseChecksRequiredJSONSchemaFields(t *testing.T) { + request := &dto.GeneralOpenAIRequest{ResponseFormat: &dto.ResponseFormat{ + Type: "json_schema", + JsonSchema: []byte(`{"name":"answer","schema":{"type":"object","required":["answer"],"properties":{"answer":{"type":"string"}}}}`), + }} + assert.Error(t, ValidateResponse(request, types.RelayFormatOpenAI, []byte(`{"choices":[{"message":{"content":"{\"other\":1}"}}]}`))) + require.NoError(t, ValidateResponse(request, types.RelayFormatOpenAI, []byte(`{"choices":[{"message":{"content":"{\"answer\":\"yes\"}"}}]}`))) +} + +func TestValidateResponseRejectsIncompleteCodeFenceForCodeTask(t *testing.T) { + request := requestWithText("write code") + assert.Error(t, ValidateResponse(request, types.RelayFormatOpenAI, []byte("{\"choices\":[{\"message\":{\"content\":\"```go\\nfunc main() {}\"}}]}"))) + require.NoError(t, ValidateResponse(request, types.RelayFormatOpenAI, []byte("{\"choices\":[{\"message\":{\"content\":\"```go\\nfunc main() {}\\n```\"}}]}"))) +} + +func TestValidateResponseChecksToolNameAndArguments(t *testing.T) { + request := &dto.GeneralOpenAIRequest{Tools: []dto.ToolCallRequest{{Type: "function", Function: dto.FunctionRequest{Name: "weather"}}}} + assert.Error(t, ValidateResponse(request, types.RelayFormatOpenAI, []byte(`{"choices":[{"message":{"tool_calls":[{"function":{"name":"unknown","arguments":"{}"}}]}}]}`))) + assert.Error(t, ValidateResponse(request, types.RelayFormatOpenAI, []byte(`{"choices":[{"message":{"tool_calls":[{"function":{"name":"weather","arguments":"bad"}}]}}]}`))) + require.NoError(t, ValidateResponse(request, types.RelayFormatOpenAI, []byte(`{"choices":[{"message":{"tool_calls":[{"function":{"name":"weather","arguments":"{}"}}]}}]}`))) +} + +func TestValidateResponseChecksToolArgumentSchema(t *testing.T) { + request := &dto.GeneralOpenAIRequest{Tools: []dto.ToolCallRequest{ + {Type: "function", Function: dto.FunctionRequest{ + Name: "weather", + Parameters: map[string]any{ + "type": "object", "required": []any{"city"}, + "properties": map[string]any{"city": map[string]any{"type": "string"}}, + }, + }}, + }} + assert.Error(t, ValidateResponse(request, types.RelayFormatOpenAI, []byte(`{"choices":[{"message":{"tool_calls":[{"function":{"name":"weather","arguments":"{}"}}]}}]}`))) + require.NoError(t, ValidateResponse(request, types.RelayFormatOpenAI, []byte(`{"choices":[{"message":{"tool_calls":[{"function":{"name":"weather","arguments":"{\"city\":\"Paris\"}"}}]}}]}`))) +} + +func TestValidateResponsesAPIRejectsIncompleteResponse(t *testing.T) { + request := &dto.OpenAIResponsesRequest{} + assert.Error(t, ValidateResponse(request, types.RelayFormatOpenAIResponses, []byte(`{"status":"incomplete","output":[]}`))) + require.NoError(t, ValidateResponse(request, types.RelayFormatOpenAIResponses, []byte(`{"status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"done"}]}]}`))) +} + +func TestValidateResponsesAPIChecksFunctionCallArguments(t *testing.T) { + request := &dto.OpenAIResponsesRequest{Tools: []byte(`[{"type":"function","name":"weather","parameters":{"type":"object"}}]`)} + assert.Error(t, ValidateResponse(request, types.RelayFormatOpenAIResponses, []byte(`{"status":"completed","output":[{"type":"function_call","name":"weather","arguments":"bad"}]}`))) + require.NoError(t, ValidateResponse(request, types.RelayFormatOpenAIResponses, []byte(`{"status":"completed","output":[{"type":"function_call","name":"weather","arguments":"{}"}]}`))) +} + +func TestValidateResponsesAPIChecksJSONSchemaOutput(t *testing.T) { + request := &dto.OpenAIResponsesRequest{Text: []byte(`{"format":{"type":"json_schema","schema":{"type":"object","required":["answer"]}}}`)} + assert.Error(t, ValidateResponse(request, types.RelayFormatOpenAIResponses, []byte(`{"status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"{\"other\":1}"}]}]}`))) + require.NoError(t, ValidateResponse(request, types.RelayFormatOpenAIResponses, []byte(`{"status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"{\"answer\":1}"}]}]}`))) +} diff --git a/service/intelligent_routing_audit_test.go b/service/intelligent_routing_audit_test.go new file mode 100644 index 000000000000..0c012e3f2169 --- /dev/null +++ b/service/intelligent_routing_audit_test.go @@ -0,0 +1,38 @@ +package service + +import ( + "net/http/httptest" + "testing" + + relaycommon "github.com/QuantumNous/new-api/relay/common" + hosttypes "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/shopspring/decimal" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateTextOtherInfoAddsAdminOnlyIntelligentRoutingAudit(t *testing.T) { + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + info := &relaycommon.RelayInfo{OriginModelName: "client-model", IntelligentRouteShadow: true, ChannelMeta: &relaycommon.ChannelMeta{}, IntelligentRoutePlan: &hosttypes.IntelligentRoutePlan{ + PolicyVersion: 3, RequestedModel: "client-model", + Nodes: []hosttypes.IntelligentRouteNode{{Model: "cheap", ChannelID: 7, PredictedSuccess: .92, ExpectedCost: decimal.RequireFromString("0.001"), ReasonCodes: []string{"lowest_expected_cost"}}}, + }, IntelligentRouteAttempts: []hosttypes.IntelligentRouteAttempt{ + {Index: 0, Model: "cheap", ChannelID: 7, Outcome: "failed", FailureReason: "timeout", LatencyMS: 120}, + {Index: 1, Model: "safe", ChannelID: 8, Outcome: "success", LatencyMS: 80}, + }} + other := GenerateTextOtherInfo(ctx, info, 1, 1, 1, 0, 0, 0, 1) + admin, ok := other["admin_info"].(map[string]interface{}) + require.True(t, ok) + audit, ok := admin["intelligent_routing"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, 3, audit["policy_version"]) + assert.Equal(t, "client-model", audit["requested_model"]) + assert.Equal(t, true, audit["shadow"]) + require.NotNil(t, audit["candidates"]) + attempts, ok := audit["attempts"].([]map[string]interface{}) + require.True(t, ok) + require.Len(t, attempts, 2) + assert.Equal(t, "timeout", attempts[0]["failure_reason"]) + assert.Equal(t, "success", attempts[1]["outcome"]) +} diff --git a/service/log_info_generate.go b/service/log_info_generate.go index 353f7098f7a1..1296b9571b72 100644 --- a/service/log_info_generate.go +++ b/service/log_info_generate.go @@ -107,6 +107,7 @@ func GenerateTextOtherInfo(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, m } AppendChannelAffinityAdminInfo(ctx, adminInfo) + appendIntelligentRoutingAdminInfo(relayInfo, adminInfo) other["admin_info"] = adminInfo appendRequestPath(ctx, relayInfo, other) @@ -118,6 +119,64 @@ func GenerateTextOtherInfo(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, m return other } +func appendIntelligentRoutingAdminInfo(relayInfo *relaycommon.RelayInfo, adminInfo map[string]interface{}) { + if relayInfo == nil || adminInfo == nil { + return + } + plan := relayInfo.IntelligentRoutePlan + if plan == nil && relayInfo.IntelligentRouteError == "" { + return + } + audit := map[string]interface{}{ + "shadow": relayInfo.IntelligentRouteShadow, + "requested_model": relayInfo.OriginModelName, + "execution_model": relayInfo.GetExecutionModelName(), + "attempt_index": relayInfo.IntelligentRouteAttempt, + } + if relayInfo.IntelligentRouteRolloutRevision > 0 { + audit["rollout_revision"] = relayInfo.IntelligentRouteRolloutRevision + audit["rollout_bucket"] = relayInfo.IntelligentRouteRolloutBucket + audit["rollout_mode"] = relayInfo.IntelligentRouteRolloutMode + audit["policy_version"] = relayInfo.IntelligentRoutePolicyVersion + } + if relayInfo.IntelligentRouteError != "" { + audit["error"] = relayInfo.IntelligentRouteError + } + if plan != nil { + audit["policy_version"] = plan.PolicyVersion + candidates := make([]map[string]interface{}, 0, len(plan.Nodes)) + for i, node := range plan.Nodes { + if i == 4 { + break + } + reasons := node.ReasonCodes + if len(reasons) > 8 { + reasons = reasons[:8] + } + candidates = append(candidates, map[string]interface{}{ + "model": node.Model, "channel_id": node.ChannelID, "tier": node.Tier, + "predicted_success": node.PredictedSuccess, "expected_cost": node.ExpectedCost.String(), + "reason_codes": reasons, + }) + } + audit["candidates"] = candidates + } + if len(relayInfo.IntelligentRouteAttempts) > 0 { + attempts := make([]map[string]interface{}, 0, min(len(relayInfo.IntelligentRouteAttempts), 4)) + for i, attempt := range relayInfo.IntelligentRouteAttempts { + if i == 4 { + break + } + attempts = append(attempts, map[string]interface{}{ + "index": attempt.Index, "model": attempt.Model, "channel_id": attempt.ChannelID, + "outcome": attempt.Outcome, "failure_reason": attempt.FailureReason, "latency_ms": attempt.LatencyMS, + }) + } + audit["attempts"] = attempts + } + adminInfo["intelligent_routing"] = audit +} + func appendParamOverrideInfo(relayInfo *relaycommon.RelayInfo, other map[string]interface{}) { if relayInfo == nil || other == nil || len(relayInfo.ParamOverrideAudit) == 0 { return diff --git a/service/text_quota.go b/service/text_quota.go index b7578f732786..69d5d7a50f51 100644 --- a/service/text_quota.go +++ b/service/text_quota.go @@ -230,7 +230,7 @@ func composeTieredTextQuota(relayInfo *relaycommon.RelayInfo, summary textQuotaS // the result with tiered billing, affinity observation and logging. func calculateTextQuotaSummary(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, usage *dto.Usage) textQuotaSummary { summary := textQuotaSummary{ - ModelName: relayInfo.OriginModelName, + ModelName: relayInfo.GetBillingModelName(), TokenName: ctx.GetString("token_name"), UseTimeSeconds: time.Now().Unix() - relayInfo.StartTime.Unix(), CompletionRatio: relayInfo.PriceData.CompletionRatio, diff --git a/setting/intelligent_routing_setting/config.go b/setting/intelligent_routing_setting/config.go new file mode 100644 index 000000000000..e777549afa73 --- /dev/null +++ b/setting/intelligent_routing_setting/config.go @@ -0,0 +1,159 @@ +package intelligent_routing_setting + +import ( + "errors" + "fmt" + "sync/atomic" + "time" + + "github.com/QuantumNous/new-api/setting/config" +) + +type TaskType string + +const ( + TaskTranslation TaskType = "translation" + TaskSummary TaskType = "summary" + TaskGeneral TaskType = "general" + TaskExtraction TaskType = "extraction" + TaskCode TaskType = "code" + TaskReasoning TaskType = "reasoning" + TaskJSON TaskType = "json_schema" + TaskTool TaskType = "tool" + + MaxPolicyDocumentBytes = 1 << 20 + MaxPolicyModels = 512 + MaxAttempts = 8 + MaxEndpointsPerModel = 4 + MaxExecutionBudget = 2 * time.Minute + MaxContextLimit = 10_000_000 + MaxModelPrice = 1_000_000 + MaxCostMultiplier = 100 +) + +type ModelPolicy struct { + Model string `json:"model"` + Tier int `json:"tier"` + InputPrice float64 `json:"input_price"` + OutputPrice float64 `json:"output_price"` + ContextLimit int `json:"context_limit"` + Capabilities []string `json:"capabilities"` +} + +type Config struct { + Enabled bool `json:"enabled"` + ShadowOnly bool `json:"shadow_only"` + PolicyVersion int `json:"policy_version"` + MaxAttempts int `json:"max_attempts"` + MaxEndpointsPerModel int `json:"max_endpoints_per_model"` + NonStreamBudget time.Duration `json:"non_stream_budget"` + StreamFirstByteBudget time.Duration `json:"stream_first_byte_budget"` + MaxCostMultiplier float64 `json:"max_cost_multiplier"` + QualityThresholds map[TaskType]float64 `json:"quality_thresholds"` + Models []ModelPolicy `json:"models"` +} + +var current atomic.Pointer[Config] +var registeredConfig Config + +func init() { + normalized, _ := Normalize(Config{}) + registeredConfig = normalized + current.Store(&normalized) + config.GlobalConfig.Register("intelligent_routing_setting", ®isteredConfig) +} + +func Normalize(input Config) (Config, error) { + if input.PolicyVersion == 0 { + input.PolicyVersion = 1 + } + if input.MaxAttempts == 0 { + input.MaxAttempts = 4 + } + if input.MaxEndpointsPerModel == 0 { + input.MaxEndpointsPerModel = 2 + } + if input.NonStreamBudget == 0 { + input.NonStreamBudget = 30 * time.Second + } + if input.StreamFirstByteBudget == 0 { + input.StreamFirstByteBudget = 12 * time.Second + } + if input.MaxCostMultiplier == 0 { + input.MaxCostMultiplier = 2.5 + } + if input.PolicyVersion < 1 || input.MaxAttempts < 1 || input.MaxAttempts > MaxAttempts || + input.MaxEndpointsPerModel < 1 || input.MaxEndpointsPerModel > MaxEndpointsPerModel || + input.NonStreamBudget < 0 || input.NonStreamBudget > MaxExecutionBudget || + input.StreamFirstByteBudget < 0 || input.StreamFirstByteBudget > MaxExecutionBudget || + input.MaxCostMultiplier < 1 || input.MaxCostMultiplier > MaxCostMultiplier { + return Config{}, errors.New("invalid intelligent routing budget") + } + defaults := map[TaskType]float64{ + TaskTranslation: .88, TaskSummary: .88, TaskGeneral: .90, TaskCode: .93, TaskExtraction: .94, + TaskReasoning: .95, TaskJSON: .97, TaskTool: .98, + } + for task, value := range input.QualityThresholds { + if value < 0 || value > 1 { + return Config{}, fmt.Errorf("quality threshold for %s must be between 0 and 1", task) + } + defaults[task] = value + } + input.QualityThresholds = defaults + if len(input.Models) > MaxPolicyModels { + return Config{}, errors.New("too many intelligent routing model policies") + } + seen := make(map[string]struct{}, len(input.Models)) + for _, policy := range input.Models { + if policy.Model == "" || policy.Tier < 0 || policy.Tier > 3 || + policy.InputPrice < 0 || policy.InputPrice > MaxModelPrice || + policy.OutputPrice < 0 || policy.OutputPrice > MaxModelPrice || + policy.ContextLimit < 0 || policy.ContextLimit > MaxContextLimit { + return Config{}, fmt.Errorf("invalid model policy for %q", policy.Model) + } + if _, ok := seen[policy.Model]; ok { + return Config{}, fmt.Errorf("duplicate model policy %q", policy.Model) + } + seen[policy.Model] = struct{}{} + } + return clone(input), nil +} + +func Update(input Config) error { + normalized, err := Normalize(input) + if err != nil { + return err + } + current.Store(&normalized) + registeredConfig = clone(normalized) + return nil +} + +func UpdateAndSync() error { + return Update(registeredConfig) +} + +func Get() Config { + value := current.Load() + if value == nil { + return Config{} + } + return clone(*value) +} + +func Enabled() bool { + return Get().Enabled +} + +func clone(input Config) Config { + input.Models = append([]ModelPolicy(nil), input.Models...) + for i := range input.Models { + input.Models[i].Capabilities = append([]string(nil), input.Models[i].Capabilities...) + } + thresholds := input.QualityThresholds + input.QualityThresholds = make(map[TaskType]float64, len(thresholds)) + for task, value := range thresholds { + input.QualityThresholds[task] = value + } + return input +} diff --git a/setting/intelligent_routing_setting/config_test.go b/setting/intelligent_routing_setting/config_test.go new file mode 100644 index 000000000000..65e4f6847631 --- /dev/null +++ b/setting/intelligent_routing_setting/config_test.go @@ -0,0 +1,48 @@ +package intelligent_routing_setting + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeConfigAppliesSafeDefaults(t *testing.T) { + got, err := Normalize(Config{Enabled: true}) + require.NoError(t, err) + assert.Equal(t, 1, got.PolicyVersion) + assert.Equal(t, 4, got.MaxAttempts) + assert.Equal(t, 2, got.MaxEndpointsPerModel) + assert.Equal(t, 30*time.Second, got.NonStreamBudget) + assert.Equal(t, 12*time.Second, got.StreamFirstByteBudget) + assert.InDelta(t, 2.5, got.MaxCostMultiplier, 0.0001) + assert.InDelta(t, 0.98, got.QualityThresholds[TaskTool], 0.0001) +} + +func TestNormalizeConfigRejectsInvalidValues(t *testing.T) { + tests := []Config{ + {MaxAttempts: -1}, + {MaxAttempts: MaxAttempts + 1}, + {MaxEndpointsPerModel: MaxEndpointsPerModel + 1}, + {NonStreamBudget: MaxExecutionBudget + time.Nanosecond}, + {StreamFirstByteBudget: MaxExecutionBudget + time.Nanosecond}, + {MaxCostMultiplier: MaxCostMultiplier + 0.01}, + {QualityThresholds: map[TaskType]float64{TaskGeneral: 1.1}}, + {Models: []ModelPolicy{{Model: "a", Tier: 4}}}, + {Models: []ModelPolicy{{Model: "a"}, {Model: "a"}}}, + } + for _, input := range tests { + _, err := Normalize(input) + assert.Error(t, err) + } +} + +func TestUpdatePublishesIndependentSnapshot(t *testing.T) { + require.NoError(t, Update(Config{Enabled: true, Models: []ModelPolicy{{Model: "cheap", Tier: 0}}})) + first := Get() + first.Models[0].Model = "changed" + second := Get() + assert.Equal(t, "cheap", second.Models[0].Model) + assert.True(t, Enabled()) +} diff --git a/types/intelligent_routing.go b/types/intelligent_routing.go new file mode 100644 index 000000000000..6ba68e8baf08 --- /dev/null +++ b/types/intelligent_routing.go @@ -0,0 +1,29 @@ +package types + +import "github.com/shopspring/decimal" + +type IntelligentRouteNode struct { + Model string + ChannelID int + Tier int + PredictedSuccess float64 + ExpectedCost decimal.Decimal + ReasonCodes []string +} + +type IntelligentRoutePlan struct { + RequestedModel string + PolicyVersion int + Nodes []IntelligentRouteNode + MaxAttempts int + MaxCostMultiplier float64 +} + +type IntelligentRouteAttempt struct { + Index int + Model string + ChannelID int + Outcome string + FailureReason string + LatencyMS int64 +} diff --git a/verification-intelligent-routing-policy-control/DIFF_FILE b/verification-intelligent-routing-policy-control/DIFF_FILE new file mode 100644 index 000000000000..4ae07e140416 --- /dev/null +++ b/verification-intelligent-routing-policy-control/DIFF_FILE @@ -0,0 +1,1654 @@ +diff --git a/controller/audit.go b/controller/audit.go +index d6974b90..fa9f0e20 100644 +--- a/controller/audit.go ++++ b/controller/audit.go +@@ -16,19 +16,24 @@ import ( + // action 的 params 填充。本地化展示文案在前端 i18n 模板中维护，本表是语言中立的 + // 英文基线——调用方因此无需在每个埋点处手写句子（避免与 params 重复书写同一份值）。 + var auditContentTemplates = map[string]string{ +- "user.create": "Created user ${username} (role ${role})", +- "user.update": "Updated user ${username} (ID: ${id})", +- "user.delete": "Deleted user ${username} (ID: ${id})", +- "user.manage": "Performed ${action} on user ${username} (ID: ${id})", +- "user.quota_add": "Increased user quota by ${quota}", +- "user.quota_subtract": "Decreased user quota by ${quota}", +- "user.quota_override": "Overrode user quota from ${from} to ${to}", +- "user.binding_clear": "Cleared ${bindingType} binding for user ${username}", +- "user.2fa_disable": "Force-disabled two-factor authentication for the user", +- "user.passkey_register": "Registered a passkey", +- "user.passkey_delete": "Deleted a passkey", +- "user.reset_passkey": "Reset the user passkey", +- "option.update": "Updated system setting ${key}", ++ "user.create": "Created user ${username} (role ${role})", ++ "user.update": "Updated user ${username} (ID: ${id})", ++ "user.delete": "Deleted user ${username} (ID: ${id})", ++ "user.manage": "Performed ${action} on user ${username} (ID: ${id})", ++ "user.quota_add": "Increased user quota by ${quota}", ++ "user.quota_subtract": "Decreased user quota by ${quota}", ++ "user.quota_override": "Overrode user quota from ${from} to ${to}", ++ "user.binding_clear": "Cleared ${bindingType} binding for user ${username}", ++ "user.2fa_disable": "Force-disabled two-factor authentication for the user", ++ "user.passkey_register": "Registered a passkey", ++ "user.passkey_delete": "Deleted a passkey", ++ "user.reset_passkey": "Reset the user passkey", ++ "option.update": "Updated system setting ${key}", ++ "intelligent_routing.policy.create": "Created intelligent routing policy ${id}", ++ "intelligent_routing.policy.update": "Updated intelligent routing policy ${id}", ++ "intelligent_routing.policy.publish": "Published intelligent routing policy ${version}", ++ "intelligent_routing.policy.rollback": "Rolled back intelligent routing policy to ${source_version} as ${version}", ++ "intelligent_routing.rollout.update": "Updated intelligent routing rollout revision ${revision}", + + "channel.create": "Created channel ${name} (type ${type}, count ${count})", + "channel.update": "Updated channel ${name} (ID: ${id})", +diff --git a/controller/intelligent_routing.go b/controller/intelligent_routing.go +new file mode 100644 +index 00000000..356ae176 +--- /dev/null ++++ b/controller/intelligent_routing.go +@@ -0,0 +1,207 @@ ++package controller ++ ++import ( ++ "errors" ++ "net/http" ++ "strconv" ++ ++ "github.com/QuantumNous/new-api/common" ++ "github.com/QuantumNous/new-api/dto" ++ "github.com/QuantumNous/new-api/model" ++ intelligentrouting "github.com/QuantumNous/new-api/service/intelligent_routing" ++ "github.com/gin-gonic/gin" ++) ++ ++func ListIntelligentRoutingPolicies(c *gin.Context) { ++ page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) ++ pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20")) ++ if page < 1 { ++ page = 1 ++ } ++ if pageSize < 1 { ++ pageSize = 20 ++ } ++ if pageSize > 100 { ++ pageSize = 100 ++ } ++ policies, total, err := model.ListIntelligentRoutingPolicies((page-1)*pageSize, pageSize) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ c.JSON(http.StatusOK, gin.H{"success": true, "data": policies, "total": total, "page": page, "page_size": pageSize}) ++} ++ ++func GetIntelligentRoutingPolicy(c *gin.Context) { ++ id, err := strconv.ParseInt(c.Param("id"), 10, 64) ++ if err != nil || id < 1 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy id"}) ++ return ++ } ++ policy, err := model.GetIntelligentRoutingPolicy(id) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ c.JSON(http.StatusOK, gin.H{"success": true, "data": policy}) ++} ++ ++func CreateIntelligentRoutingPolicy(c *gin.Context) { ++ var request dto.IntelligentRoutingDraftRequest ++ if err := common.DecodeJson(c.Request.Body, &request); err != nil { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) ++ return ++ } ++ policy, issues, err := intelligentrouting.DefaultPolicyControl.CreateDraft(c, request.Config, c.GetInt("id")) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ if len(issues) > 0 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) ++ return ++ } ++ recordManageAudit(c, "intelligent_routing.policy.create", map[string]interface{}{"id": policy.Id}) ++ c.JSON(http.StatusCreated, gin.H{"success": true, "data": policy}) ++} ++ ++func UpdateIntelligentRoutingPolicy(c *gin.Context) { ++ id, err := strconv.ParseInt(c.Param("id"), 10, 64) ++ if err != nil || id < 1 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy id"}) ++ return ++ } ++ var request dto.IntelligentRoutingDraftUpdateRequest ++ if err := common.DecodeJson(c.Request.Body, &request); err != nil { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) ++ return ++ } ++ policy, issues, err := intelligentrouting.DefaultPolicyControl.UpdateDraft(c, id, request.UpdatedAt, request.Config) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ if len(issues) > 0 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) ++ return ++ } ++ recordManageAudit(c, "intelligent_routing.policy.update", map[string]interface{}{"id": policy.Id}) ++ c.JSON(http.StatusOK, gin.H{"success": true, "data": policy}) ++} ++ ++func ValidateIntelligentRoutingPolicy(c *gin.Context) { ++ id, err := strconv.ParseInt(c.Param("id"), 10, 64) ++ if err != nil || id < 1 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy id"}) ++ return ++ } ++ policy, err := model.GetIntelligentRoutingPolicy(id) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ validated, issues := intelligentrouting.ValidatePolicyDocument(policy.Config) ++ c.JSON(http.StatusOK, gin.H{"success": len(issues) == 0, "data": gin.H{"checksum": validated.Checksum, "issues": issues}}) ++} ++ ++func PublishIntelligentRoutingPolicy(c *gin.Context) { ++ id, err := strconv.ParseInt(c.Param("id"), 10, 64) ++ if err != nil || id < 1 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy id"}) ++ return ++ } ++ var request dto.IntelligentRoutingPublishRequest ++ if err := common.DecodeJson(c.Request.Body, &request); err != nil { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) ++ return ++ } ++ policy, issues, err := intelligentrouting.DefaultPolicyControl.Publish(c, id, c.GetInt("id"), request.ChangeNote) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ if len(issues) > 0 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) ++ return ++ } ++ recordManageAudit(c, "intelligent_routing.policy.publish", map[string]interface{}{"id": policy.Id, "version": policy.Version}) ++ c.JSON(http.StatusOK, gin.H{"success": true, "data": policy}) ++} ++ ++func RollbackIntelligentRoutingPolicy(c *gin.Context) { ++ version, err := strconv.Atoi(c.Param("version")) ++ if err != nil || version < 1 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid policy version"}) ++ return ++ } ++ var request dto.IntelligentRoutingPublishRequest ++ if err := common.DecodeJson(c.Request.Body, &request); err != nil { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) ++ return ++ } ++ policy, issues, err := intelligentrouting.DefaultPolicyControl.Rollback(c, version, c.GetInt("id"), request.ChangeNote) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ if len(issues) > 0 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) ++ return ++ } ++ recordManageAudit(c, "intelligent_routing.policy.rollback", map[string]interface{}{"source_version": version, "version": policy.Version}) ++ c.JSON(http.StatusOK, gin.H{"success": true, "data": policy}) ++} ++ ++func GetIntelligentRoutingRollout(c *gin.Context) { ++ rollout, err := model.GetIntelligentRoutingRollout() ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ c.JSON(http.StatusOK, gin.H{"success": true, "data": rollout}) ++} ++ ++func UpdateIntelligentRoutingRollout(c *gin.Context) { ++ var request dto.IntelligentRoutingRolloutUpdateRequest ++ if err := common.DecodeJson(c.Request.Body, &request); err != nil { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "invalid request"}) ++ return ++ } ++ userGroups, err := common.Marshal(request.UserGroups) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ tokenGroups, err := common.Marshal(request.TokenGroups) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ rollout, issues, err := intelligentrouting.DefaultPolicyControl.UpdateRollout(c, request.Revision, model.IntelligentRoutingRollout{ ++ PolicyVersion: request.PolicyVersion, Enabled: request.Enabled, Mode: request.Mode, TrafficPercent: request.TrafficPercent, ++ UserGroups: string(userGroups), TokenGroups: string(tokenGroups), UpdatedBy: c.GetInt("id"), ++ }) ++ if err != nil { ++ intelligentRoutingError(c, err) ++ return ++ } ++ if len(issues) > 0 { ++ c.JSON(http.StatusBadRequest, gin.H{"success": false, "issues": issues}) ++ return ++ } ++ recordManageAudit(c, "intelligent_routing.rollout.update", map[string]interface{}{"revision": rollout.Revision, "policy_version": rollout.PolicyVersion, "mode": rollout.Mode, "traffic_percent": rollout.TrafficPercent}) ++ c.JSON(http.StatusOK, gin.H{"success": true, "data": rollout}) ++} ++ ++func intelligentRoutingError(c *gin.Context, err error) { ++ switch { ++ case errors.Is(err, model.ErrIntelligentRoutingPolicyNotFound), errors.Is(err, model.ErrIntelligentRoutingRolloutNotFound): ++ c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "resource not found"}) ++ case errors.Is(err, model.ErrIntelligentRoutingRevisionConflict): ++ c.JSON(http.StatusConflict, gin.H{"success": false, "message": "revision conflict"}) ++ case errors.Is(err, model.ErrIntelligentRoutingPolicyImmutable): ++ c.JSON(http.StatusConflict, gin.H{"success": false, "message": "published policy is immutable"}) ++ default: ++ c.JSON(http.StatusServiceUnavailable, gin.H{"success": false, "message": "intelligent routing service unavailable"}) ++ } ++} +diff --git a/controller/relay.go b/controller/relay.go +index 6d4236fe..79dc7909 100644 +--- a/controller/relay.go ++++ b/controller/relay.go +@@ -157,9 +157,25 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { + + relayInfo.SetEstimatePromptTokens(tokens) + routingConfig := routingsetting.Get() ++ runtimeSnapshot := intelligentrouting.DefaultPolicyControl.Snapshot() ++ if runtimeSnapshot.Rollout.Exists { ++ decision := intelligentrouting.ResolveRollout(runtimeSnapshot, intelligentrouting.RolloutSubject{ ++ AccountID: relayInfo.UserId, ++ TokenID: relayInfo.TokenId, ++ UserGroup: relayInfo.UserGroup, ++ TokenGroup: relayInfo.TokenGroup, ++ }) ++ routingConfig = runtimeSnapshot.Config ++ routingConfig.Enabled = decision.Selected ++ routingConfig.ShadowOnly = decision.Mode == model.IntelligentRoutingModeShadow ++ relayInfo.IntelligentRoutePolicyVersion = decision.PolicyVersion ++ relayInfo.IntelligentRouteRolloutRevision = decision.Revision ++ relayInfo.IntelligentRouteRolloutBucket = decision.Bucket ++ relayInfo.IntelligentRouteRolloutMode = decision.Mode ++ } + intelligentRoutingActive := routingConfig.Enabled && supportsIntelligentRouting(relayInfo.RelayFormat, relayInfo.RelayMode) + if intelligentRoutingActive { +- if routingErr := buildShadowRoutePlan(c, relayInfo, tokens, nil); routingErr != nil { ++ if routingErr := buildRoutePlan(c, relayInfo, tokens, nil, routingConfig); routingErr != nil { + relayInfo.IntelligentRouteError = routingErr.Error() + if routingConfig.ShadowOnly { + logger.LogWarn(c, "intelligent routing shadow plan failed: "+routingErr.Error()) +@@ -386,8 +402,11 @@ func recordIntelligentRouteAttempt(info *relaycommon.RelayInfo, node hosttypes.I + } + + func buildShadowRoutePlan(c *gin.Context, info *relaycommon.RelayInfo, promptTokens int, source intelligentrouting.ChannelSource) error { ++ return buildRoutePlan(c, info, promptTokens, source, routingsetting.Get()) ++} ++ ++func buildRoutePlan(c *gin.Context, info *relaycommon.RelayInfo, promptTokens int, source intelligentrouting.ChannelSource, config routingsetting.Config) error { + startedAt := time.Now() +- config := routingsetting.Get() + features := intelligentrouting.ExtractFeatures(intelligentrouting.Input{ + Request: info.Request, RelayFormat: info.RelayFormat, PromptTokens: promptTokens, RequestPath: c.Request.URL.Path, + }) +@@ -412,6 +431,7 @@ func buildShadowRoutePlan(c *gin.Context, info *relaycommon.RelayInfo, promptTok + } + info.IntelligentRoutePlan = &plan + info.IntelligentRouteShadow = config.ShadowOnly ++ info.IntelligentRouteLive = !config.ShadowOnly + info.IntelligentRouteSessionKey = sessionKey + info.IntelligentRouteTask = string(features.Task) + intelligentrouting.DefaultMetrics.Observe(intelligentrouting.Observation{CandidateTier: plan.Nodes[0].Tier, PlanningDuration: time.Since(startedAt)}) +@@ -486,8 +506,7 @@ func fastTokenCountMetaForPricing(request dto.Request) *types.TokenCountMeta { + } + + func getChannel(c *gin.Context, info *relaycommon.RelayInfo, retryParam *service.RetryParam) (*model.Channel, *types.NewAPIError) { +- routingConfig := routingsetting.Get() +- if routingConfig.Enabled && !routingConfig.ShadowOnly && info.IntelligentRoutePlan != nil { ++ if info.IntelligentRouteLive && info.IntelligentRoutePlan != nil { + index := retryParam.GetRetry() + if index < 0 || index >= len(info.IntelligentRoutePlan.Nodes) { + return nil, types.NewError(errors.New("intelligent route attempts exhausted"), types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) +diff --git a/docs/intelligent-routing-shadow-rollout.md b/docs/intelligent-routing-shadow-rollout.md +index e4ea49ae..27fd006a 100644 +--- a/docs/intelligent-routing-shadow-rollout.md ++++ b/docs/intelligent-routing-shadow-rollout.md +@@ -10,6 +10,10 @@ Routing details remain backend-only under `other.admin_info.intelligent_routing` + + ## Configuration + ++Published administrator policies and scoped rollouts are managed through the root-only `/api/intelligent-routing` endpoints. Drafts are validated and checksummed before publication; published versions are immutable, and rollback creates a new version. A rollout selects a published version, `shadow` or `live` mode, user/token group allowlists, and a deterministic traffic percentage guarded by a revision number. ++ ++Instances refresh the durable rollout snapshot at startup and periodically. The stable bucket uses the policy version, account, and token so the same caller remains in the same cohort. If no durable rollout exists, the legacy global configuration below remains active for backward compatibility. ++ + Configure the registered `intelligent_routing_setting` object through the system options API or administrator settings storage: + + - `enabled`: enables planning. +diff --git a/dto/intelligent_routing.go b/dto/intelligent_routing.go +new file mode 100644 +index 00000000..275d53ef +--- /dev/null ++++ b/dto/intelligent_routing.go +@@ -0,0 +1,26 @@ ++package dto ++ ++import "time" ++ ++type IntelligentRoutingDraftRequest struct { ++ Config string `json:"config"` ++} ++ ++type IntelligentRoutingDraftUpdateRequest struct { ++ Config string `json:"config"` ++ UpdatedAt time.Time `json:"updated_at"` ++} ++ ++type IntelligentRoutingPublishRequest struct { ++ ChangeNote string `json:"change_note"` ++} ++ ++type IntelligentRoutingRolloutUpdateRequest struct { ++ Revision int64 `json:"revision"` ++ PolicyVersion int `json:"policy_version"` ++ Enabled bool `json:"enabled"` ++ Mode string `json:"mode"` ++ TrafficPercent int `json:"traffic_percent"` ++ UserGroups []string `json:"user_groups"` ++ TokenGroups []string `json:"token_groups"` ++} +diff --git a/main.go b/main.go +index 742d1551..711eeffe 100644 +--- a/main.go ++++ b/main.go +@@ -29,6 +29,7 @@ import ( + "github.com/QuantumNous/new-api/router" + "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/service/authz" ++ intelligentrouting "github.com/QuantumNous/new-api/service/intelligent_routing" + _ "github.com/QuantumNous/new-api/setting/performance_setting" + "github.com/QuantumNous/new-api/setting/ratio_setting" + +@@ -338,6 +339,10 @@ func InitResources() error { + if err != nil { + return err + } ++ if err := intelligentrouting.DefaultPolicyControl.RefreshSnapshot(context.Background()); err != nil { ++ common.SysError("failed to load intelligent routing policy snapshot: " + err.Error()) ++ } ++ intelligentrouting.StartPolicyRefresh(context.Background(), intelligentrouting.DefaultPolicyControl, time.Duration(common.SyncFrequency)*time.Second) + + perfmetrics.Init() + +diff --git a/model/intelligent_routing_policy.go b/model/intelligent_routing_policy.go +new file mode 100644 +index 00000000..62b1434b +--- /dev/null ++++ b/model/intelligent_routing_policy.go +@@ -0,0 +1,252 @@ ++package model ++ ++import ( ++ "errors" ++ "time" ++ ++ "gorm.io/gorm" ++) ++ ++const ( ++ IntelligentRoutingPolicyDraft = "draft" ++ IntelligentRoutingPolicyActive = "active" ++ IntelligentRoutingPolicyArchived = "archived" ++ ++ IntelligentRoutingModeShadow = "shadow" ++ IntelligentRoutingModeLive = "live" ++) ++ ++var ( ++ ErrIntelligentRoutingPolicyNotFound = errors.New("intelligent routing policy not found") ++ ErrIntelligentRoutingRolloutNotFound = errors.New("intelligent routing rollout not found") ++ ErrIntelligentRoutingPolicyImmutable = errors.New("published intelligent routing policy is immutable") ++ ErrIntelligentRoutingRevisionConflict = errors.New("intelligent routing revision conflict") ++) ++ ++type IntelligentRoutingPolicy struct { ++ Id int64 `json:"id" gorm:"primaryKey"` ++ Version int `json:"version" gorm:"index"` ++ Status string `json:"status" gorm:"type:varchar(16);index"` ++ Config string `json:"config" gorm:"type:text"` ++ Checksum string `json:"checksum" gorm:"type:varchar(64)"` ++ SourceVersion int `json:"source_version"` ++ ChangeNote string `json:"change_note" gorm:"type:varchar(500)"` ++ CreatedBy int `json:"created_by"` ++ PublishedBy int `json:"published_by"` ++ PublishedAt *time.Time `json:"published_at"` ++ CreatedAt time.Time `json:"created_at"` ++ UpdatedAt time.Time `json:"updated_at"` ++} ++ ++type IntelligentRoutingRollout struct { ++ Id int64 `json:"id" gorm:"primaryKey"` ++ Revision int64 `json:"revision"` ++ PolicyVersion int `json:"policy_version"` ++ Enabled bool `json:"enabled"` ++ Mode string `json:"mode" gorm:"type:varchar(16)"` ++ TrafficPercent int `json:"traffic_percent"` ++ UserGroups string `json:"user_groups" gorm:"type:text"` ++ TokenGroups string `json:"token_groups" gorm:"type:text"` ++ UpdatedBy int `json:"updated_by"` ++ StartedAt *time.Time `json:"started_at"` ++ EndedAt *time.Time `json:"ended_at"` ++ CreatedAt time.Time `json:"created_at"` ++ UpdatedAt time.Time `json:"updated_at"` ++} ++ ++func CreateIntelligentRoutingDraft(policy IntelligentRoutingPolicy) (IntelligentRoutingPolicy, error) { ++ policy.Id = 0 ++ policy.Version = 0 ++ policy.Status = IntelligentRoutingPolicyDraft ++ policy.PublishedBy = 0 ++ policy.PublishedAt = nil ++ err := DB.Create(&policy).Error ++ return policy, err ++} ++ ++func UpdateIntelligentRoutingDraft(id int64, updatedAt time.Time, config, checksum string) (IntelligentRoutingPolicy, error) { ++ var policy IntelligentRoutingPolicy ++ err := DB.Transaction(func(tx *gorm.DB) error { ++ if err := lockForUpdate(tx).Where("id = ?", id).First(&policy).Error; err != nil { ++ if errors.Is(err, gorm.ErrRecordNotFound) { ++ return ErrIntelligentRoutingPolicyNotFound ++ } ++ return err ++ } ++ if policy.Status != IntelligentRoutingPolicyDraft { ++ return ErrIntelligentRoutingPolicyImmutable ++ } ++ result := tx.Model(&IntelligentRoutingPolicy{}). ++ Where("id = ? AND status = ? AND updated_at = ?", id, IntelligentRoutingPolicyDraft, updatedAt). ++ Updates(map[string]any{"config": config, "checksum": checksum}) ++ if result.Error != nil { ++ return result.Error ++ } ++ if result.RowsAffected != 1 { ++ return ErrIntelligentRoutingRevisionConflict ++ } ++ return tx.Where("id = ?", id).First(&policy).Error ++ }) ++ return policy, err ++} ++ ++func ListIntelligentRoutingPolicies(offset, limit int) ([]IntelligentRoutingPolicy, int64, error) { ++ var policies []IntelligentRoutingPolicy ++ var total int64 ++ if err := DB.Model(&IntelligentRoutingPolicy{}).Count(&total).Error; err != nil { ++ return nil, 0, err ++ } ++ err := DB.Order("id DESC").Offset(offset).Limit(limit).Find(&policies).Error ++ return policies, total, err ++} ++ ++func GetIntelligentRoutingPolicy(id int64) (IntelligentRoutingPolicy, error) { ++ var policy IntelligentRoutingPolicy ++ err := DB.Where("id = ?", id).First(&policy).Error ++ if errors.Is(err, gorm.ErrRecordNotFound) { ++ err = ErrIntelligentRoutingPolicyNotFound ++ } ++ return policy, err ++} ++ ++func GetActiveIntelligentRoutingPolicy() (IntelligentRoutingPolicy, error) { ++ var policy IntelligentRoutingPolicy ++ err := DB.Where("status = ?", IntelligentRoutingPolicyActive).Order("version DESC").First(&policy).Error ++ if errors.Is(err, gorm.ErrRecordNotFound) { ++ err = ErrIntelligentRoutingPolicyNotFound ++ } ++ return policy, err ++} ++ ++func GetIntelligentRoutingPolicyByVersion(version int) (IntelligentRoutingPolicy, error) { ++ var policy IntelligentRoutingPolicy ++ err := DB.Where("version = ?", version).First(&policy).Error ++ if errors.Is(err, gorm.ErrRecordNotFound) { ++ err = ErrIntelligentRoutingPolicyNotFound ++ } ++ return policy, err ++} ++ ++func PublishIntelligentRoutingPolicy(id int64, administratorID int, changeNote string) (IntelligentRoutingPolicy, error) { ++ var published IntelligentRoutingPolicy ++ err := DB.Transaction(func(tx *gorm.DB) error { ++ var draft IntelligentRoutingPolicy ++ if err := lockForUpdate(tx).Where("id = ?", id).First(&draft).Error; err != nil { ++ if errors.Is(err, gorm.ErrRecordNotFound) { ++ return ErrIntelligentRoutingPolicyNotFound ++ } ++ return err ++ } ++ if draft.Status != IntelligentRoutingPolicyDraft { ++ return ErrIntelligentRoutingPolicyImmutable ++ } ++ ++ var latest IntelligentRoutingPolicy ++ latestErr := lockForUpdate(tx).Order("version DESC").First(&latest).Error ++ if latestErr != nil && !errors.Is(latestErr, gorm.ErrRecordNotFound) { ++ return latestErr ++ } ++ nextVersion := latest.Version + 1 ++ if err := tx.Model(&IntelligentRoutingPolicy{}). ++ Where("status = ?", IntelligentRoutingPolicyActive). ++ Update("status", IntelligentRoutingPolicyArchived).Error; err != nil { ++ return err ++ } ++ now := time.Now() ++ updates := map[string]any{ ++ "version": nextVersion, "status": IntelligentRoutingPolicyActive, "change_note": changeNote, ++ "published_by": administratorID, "published_at": &now, ++ } ++ result := tx.Model(&IntelligentRoutingPolicy{}). ++ Where("id = ? AND status = ?", id, IntelligentRoutingPolicyDraft). ++ Updates(updates) ++ if result.Error != nil { ++ return result.Error ++ } ++ if result.RowsAffected != 1 { ++ return ErrIntelligentRoutingRevisionConflict ++ } ++ return tx.Where("id = ?", id).First(&published).Error ++ }) ++ return published, err ++} ++ ++func RollbackIntelligentRoutingPolicy(sourceVersion int, administratorID int, changeNote string) (IntelligentRoutingPolicy, error) { ++ var rolledBack IntelligentRoutingPolicy ++ err := DB.Transaction(func(tx *gorm.DB) error { ++ var source IntelligentRoutingPolicy ++ if err := lockForUpdate(tx).Where("version = ?", sourceVersion).First(&source).Error; err != nil { ++ if errors.Is(err, gorm.ErrRecordNotFound) { ++ return ErrIntelligentRoutingPolicyNotFound ++ } ++ return err ++ } ++ var latest IntelligentRoutingPolicy ++ latestErr := lockForUpdate(tx).Order("version DESC").First(&latest).Error ++ if latestErr != nil && !errors.Is(latestErr, gorm.ErrRecordNotFound) { ++ return latestErr ++ } ++ if err := tx.Model(&IntelligentRoutingPolicy{}). ++ Where("status = ?", IntelligentRoutingPolicyActive). ++ Update("status", IntelligentRoutingPolicyArchived).Error; err != nil { ++ return err ++ } ++ now := time.Now() ++ rolledBack = IntelligentRoutingPolicy{ ++ Version: latest.Version + 1, Status: IntelligentRoutingPolicyActive, Config: source.Config, ++ Checksum: source.Checksum, SourceVersion: sourceVersion, ChangeNote: changeNote, ++ CreatedBy: administratorID, PublishedBy: administratorID, PublishedAt: &now, ++ } ++ return tx.Create(&rolledBack).Error ++ }) ++ return rolledBack, err ++} ++ ++func GetIntelligentRoutingRollout() (IntelligentRoutingRollout, error) { ++ var rollout IntelligentRoutingRollout ++ err := DB.First(&rollout, 1).Error ++ if errors.Is(err, gorm.ErrRecordNotFound) { ++ err = ErrIntelligentRoutingRolloutNotFound ++ } ++ return rollout, err ++} ++ ++func UpdateIntelligentRoutingRollout(expectedRevision int64, next IntelligentRoutingRollout) (IntelligentRoutingRollout, error) { ++ var stored IntelligentRoutingRollout ++ err := DB.Transaction(func(tx *gorm.DB) error { ++ var current IntelligentRoutingRollout ++ err := lockForUpdate(tx).First(¤t, 1).Error ++ if errors.Is(err, gorm.ErrRecordNotFound) { ++ if expectedRevision != 0 { ++ return ErrIntelligentRoutingRevisionConflict ++ } ++ next.Id = 1 ++ next.Revision = 1 ++ if err := tx.Create(&next).Error; err != nil { ++ return err ++ } ++ stored = next ++ return nil ++ } ++ if err != nil { ++ return err ++ } ++ if current.Revision != expectedRevision { ++ return ErrIntelligentRoutingRevisionConflict ++ } ++ updates := map[string]any{ ++ "revision": expectedRevision + 1, "policy_version": next.PolicyVersion, "enabled": next.Enabled, ++ "mode": next.Mode, "traffic_percent": next.TrafficPercent, "user_groups": next.UserGroups, ++ "token_groups": next.TokenGroups, "updated_by": next.UpdatedBy, "started_at": next.StartedAt, "ended_at": next.EndedAt, ++ } ++ result := tx.Model(&IntelligentRoutingRollout{}).Where("id = ? AND revision = ?", current.Id, expectedRevision).Updates(updates) ++ if result.Error != nil { ++ return result.Error ++ } ++ if result.RowsAffected != 1 { ++ return ErrIntelligentRoutingRevisionConflict ++ } ++ return tx.First(&stored, current.Id).Error ++ }) ++ return stored, err ++} +diff --git a/model/intelligent_routing_policy_test.go b/model/intelligent_routing_policy_test.go +new file mode 100644 +index 00000000..1821fd42 +--- /dev/null ++++ b/model/intelligent_routing_policy_test.go +@@ -0,0 +1,94 @@ ++package model ++ ++import ( ++ "errors" ++ "testing" ++ ++ "github.com/stretchr/testify/assert" ++ "github.com/stretchr/testify/require" ++) ++ ++func resetIntelligentRoutingPolicyTables(t *testing.T) { ++ t.Helper() ++ require.NoError(t, DB.AutoMigrate(&IntelligentRoutingPolicy{}, &IntelligentRoutingRollout{})) ++ require.NoError(t, DB.Exec("DELETE FROM intelligent_routing_rollouts").Error) ++ require.NoError(t, DB.Exec("DELETE FROM intelligent_routing_policies").Error) ++ t.Cleanup(func() { ++ require.NoError(t, DB.Exec("DELETE FROM intelligent_routing_rollouts").Error) ++ require.NoError(t, DB.Exec("DELETE FROM intelligent_routing_policies").Error) ++ }) ++} ++ ++func TestIntelligentRoutingPolicyDraftLifecycle(t *testing.T) { ++ resetIntelligentRoutingPolicyTables(t) ++ ++ draft, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{ ++ Status: IntelligentRoutingPolicyDraft, Config: `{"enabled":false}`, Checksum: "sum", CreatedBy: 11, ++ }) ++ require.NoError(t, err) ++ assert.Equal(t, IntelligentRoutingPolicyDraft, draft.Status) ++ ++ stored, err := GetIntelligentRoutingPolicy(draft.Id) ++ require.NoError(t, err) ++ assert.Equal(t, draft.Id, stored.Id) ++ ++ stored.Status = IntelligentRoutingPolicyActive ++ require.NoError(t, DB.Save(&stored).Error) ++ _, err = UpdateIntelligentRoutingDraft(stored.Id, stored.UpdatedAt, `{"enabled":true}`, "next") ++ assert.ErrorIs(t, err, ErrIntelligentRoutingPolicyImmutable) ++} ++ ++func TestIntelligentRoutingPolicyPublishArchivesPriorVersion(t *testing.T) { ++ resetIntelligentRoutingPolicyTables(t) ++ ++ first, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{Config: `{"enabled":false}`, Checksum: "one", CreatedBy: 1}) ++ require.NoError(t, err) ++ first, err = PublishIntelligentRoutingPolicy(first.Id, 1, "first") ++ require.NoError(t, err) ++ assert.Equal(t, 1, first.Version) ++ assert.Equal(t, IntelligentRoutingPolicyActive, first.Status) ++ ++ second, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{Config: `{"enabled":true}`, Checksum: "two", CreatedBy: 2}) ++ require.NoError(t, err) ++ second, err = PublishIntelligentRoutingPolicy(second.Id, 2, "second") ++ require.NoError(t, err) ++ assert.Equal(t, 2, second.Version) ++ ++ first, err = GetIntelligentRoutingPolicy(first.Id) ++ require.NoError(t, err) ++ assert.Equal(t, IntelligentRoutingPolicyArchived, first.Status) ++} ++ ++func TestIntelligentRoutingPolicyRollbackCreatesNewVersion(t *testing.T) { ++ resetIntelligentRoutingPolicyTables(t) ++ ++ first, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{Config: `{"enabled":false}`, Checksum: "one", CreatedBy: 1}) ++ require.NoError(t, err) ++ _, err = PublishIntelligentRoutingPolicy(first.Id, 1, "first") ++ require.NoError(t, err) ++ second, err := CreateIntelligentRoutingDraft(IntelligentRoutingPolicy{Config: `{"enabled":true}`, Checksum: "two", CreatedBy: 2}) ++ require.NoError(t, err) ++ _, err = PublishIntelligentRoutingPolicy(second.Id, 2, "second") ++ require.NoError(t, err) ++ ++ rolledBack, err := RollbackIntelligentRoutingPolicy(1, 3, "restore first") ++ require.NoError(t, err) ++ assert.Equal(t, 3, rolledBack.Version) ++ assert.Equal(t, 1, rolledBack.SourceVersion) ++ assert.Equal(t, `{"enabled":false}`, rolledBack.Config) ++} ++ ++func TestIntelligentRoutingRolloutRejectsStaleRevision(t *testing.T) { ++ resetIntelligentRoutingPolicyTables(t) ++ ++ rollout, err := UpdateIntelligentRoutingRollout(0, IntelligentRoutingRollout{PolicyVersion: 1, Enabled: true, Mode: IntelligentRoutingModeShadow, TrafficPercent: 25}) ++ require.NoError(t, err) ++ assert.Equal(t, int64(1), rollout.Revision) ++ ++ _, err = UpdateIntelligentRoutingRollout(0, IntelligentRoutingRollout{PolicyVersion: 1, Enabled: false, Mode: IntelligentRoutingModeShadow}) ++ assert.True(t, errors.Is(err, ErrIntelligentRoutingRevisionConflict)) ++ ++ stored, err := GetIntelligentRoutingRollout() ++ require.NoError(t, err) ++ assert.True(t, stored.Enabled) ++} +diff --git a/model/main.go b/model/main.go +index 21445593..4dfda76f 100644 +--- a/model/main.go ++++ b/model/main.go +@@ -290,6 +290,8 @@ func migrateDB() error { + &SystemInstance{}, + &SystemTask{}, + &SystemTaskLock{}, ++ &IntelligentRoutingPolicy{}, ++ &IntelligentRoutingRollout{}, + &CasbinRule{}, + &AuthzRole{}, + ) +@@ -353,6 +355,8 @@ func migrateDBFast() error { + {&SystemInstance{}, "SystemInstance"}, + {&SystemTask{}, "SystemTask"}, + {&SystemTaskLock{}, "SystemTaskLock"}, ++ {&IntelligentRoutingPolicy{}, "IntelligentRoutingPolicy"}, ++ {&IntelligentRoutingRollout{}, "IntelligentRoutingRollout"}, + } + // 动态计算migration数量，确保errChan缓冲区足够大 + errChan := make(chan error, len(migrations)) +diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go +index 87fb91d4..215a45d1 100644 +--- a/relay/common/relay_info.go ++++ b/relay/common/relay_info.go +@@ -150,14 +150,19 @@ type RelayInfo struct { + UseRuntimeHeadersOverride bool + ParamOverrideAudit []string + +- PriceData hosttypes.PriceData +- IntelligentRoutePlan *hosttypes.IntelligentRoutePlan +- IntelligentRouteError string +- IntelligentRouteShadow bool +- IntelligentRouteAttempt int +- IntelligentRouteSessionKey string +- IntelligentRouteTask string +- IntelligentRouteAttempts []hosttypes.IntelligentRouteAttempt ++ PriceData hosttypes.PriceData ++ IntelligentRoutePlan *hosttypes.IntelligentRoutePlan ++ IntelligentRouteError string ++ IntelligentRouteShadow bool ++ IntelligentRouteLive bool ++ IntelligentRoutePolicyVersion int ++ IntelligentRouteRolloutRevision int64 ++ IntelligentRouteRolloutBucket int ++ IntelligentRouteRolloutMode string ++ IntelligentRouteAttempt int ++ IntelligentRouteSessionKey string ++ IntelligentRouteTask string ++ IntelligentRouteAttempts []hosttypes.IntelligentRouteAttempt + + // QuotaClamp is set (non-nil) when a quota conversion saturated at the + // int32 bound (or NaN fallback) while computing this request's charge. +diff --git a/router/api-router.go b/router/api-router.go +index 31c595e0..c4ef77f3 100644 +--- a/router/api-router.go ++++ b/router/api-router.go +@@ -203,6 +203,19 @@ func SetApiRouter(router *gin.Engine) { + optionRoute.POST("/waffo-pancake/subscription-product", controller.CreateWaffoPancakeSubscriptionProduct) + optionRoute.GET("/waffo-pancake/subscription-product-options", controller.ListWaffoPancakeSubscriptionProductOptions) + } ++ intelligentRoutingRoute := apiRouter.Group("/intelligent-routing") ++ intelligentRoutingRoute.Use(middleware.RootAuth()) ++ { ++ intelligentRoutingRoute.GET("/policies", controller.ListIntelligentRoutingPolicies) ++ intelligentRoutingRoute.GET("/policies/:id", controller.GetIntelligentRoutingPolicy) ++ intelligentRoutingRoute.POST("/policies", controller.CreateIntelligentRoutingPolicy) ++ intelligentRoutingRoute.PUT("/policies/:id", controller.UpdateIntelligentRoutingPolicy) ++ intelligentRoutingRoute.POST("/policies/:id/validate", controller.ValidateIntelligentRoutingPolicy) ++ intelligentRoutingRoute.POST("/policies/:id/publish", controller.PublishIntelligentRoutingPolicy) ++ intelligentRoutingRoute.POST("/policies/versions/:version/rollback", controller.RollbackIntelligentRoutingPolicy) ++ intelligentRoutingRoute.GET("/rollout", controller.GetIntelligentRoutingRollout) ++ intelligentRoutingRoute.PUT("/rollout", controller.UpdateIntelligentRoutingRollout) ++ } + + // Custom OAuth provider management (root only) + customOAuthRoute := apiRouter.Group("/custom-oauth-provider") +diff --git a/router/intelligent_routing_routes_test.go b/router/intelligent_routing_routes_test.go +new file mode 100644 +index 00000000..453d7d62 +--- /dev/null ++++ b/router/intelligent_routing_routes_test.go +@@ -0,0 +1,35 @@ ++package router ++ ++import ( ++ "net/http" ++ "testing" ++ ++ "github.com/gin-gonic/gin" ++ "github.com/stretchr/testify/assert" ++) ++ ++func TestIntelligentRoutingAdminRoutes(t *testing.T) { ++ gin.SetMode(gin.TestMode) ++ engine := gin.New() ++ SetApiRouter(engine) ++ routes := make(map[string]struct{}, len(engine.Routes())) ++ for _, route := range engine.Routes() { ++ routes[route.Method+" "+route.Path] = struct{}{} ++ } ++ ++ expected := []string{ ++ http.MethodGet + " /api/intelligent-routing/policies", ++ http.MethodGet + " /api/intelligent-routing/policies/:id", ++ http.MethodPost + " /api/intelligent-routing/policies", ++ http.MethodPut + " /api/intelligent-routing/policies/:id", ++ http.MethodPost + " /api/intelligent-routing/policies/:id/validate", ++ http.MethodPost + " /api/intelligent-routing/policies/:id/publish", ++ http.MethodPost + " /api/intelligent-routing/policies/versions/:version/rollback", ++ http.MethodGet + " /api/intelligent-routing/rollout", ++ http.MethodPut + " /api/intelligent-routing/rollout", ++ } ++ for _, route := range expected { ++ _, ok := routes[route] ++ assert.True(t, ok, route) ++ } ++} +diff --git a/service/intelligent_routing/policy_control.go b/service/intelligent_routing/policy_control.go +new file mode 100644 +index 00000000..db7f8848 +--- /dev/null ++++ b/service/intelligent_routing/policy_control.go +@@ -0,0 +1,251 @@ ++package intelligent_routing ++ ++import ( ++ "context" ++ "errors" ++ "strings" ++ "sync/atomic" ++ "time" ++ ++ "github.com/QuantumNous/new-api/common" ++ "github.com/QuantumNous/new-api/model" ++ routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" ++) ++ ++type PolicyRepository interface { ++ CreateDraft(policy model.IntelligentRoutingPolicy) (model.IntelligentRoutingPolicy, error) ++ UpdateDraft(id int64, updatedAt time.Time, config, checksum string) (model.IntelligentRoutingPolicy, error) ++ GetPolicy(id int64) (model.IntelligentRoutingPolicy, error) ++ GetPolicyByVersion(version int) (model.IntelligentRoutingPolicy, error) ++ Publish(id int64, administratorID int, note string) (model.IntelligentRoutingPolicy, error) ++ Rollback(version int, administratorID int, note string) (model.IntelligentRoutingPolicy, error) ++ GetRollout() (model.IntelligentRoutingRollout, error) ++ UpdateRollout(revision int64, rollout model.IntelligentRoutingRollout) (model.IntelligentRoutingRollout, error) ++} ++ ++type DatabasePolicyRepository struct{} ++ ++func (DatabasePolicyRepository) CreateDraft(policy model.IntelligentRoutingPolicy) (model.IntelligentRoutingPolicy, error) { ++ return model.CreateIntelligentRoutingDraft(policy) ++} ++ ++func (DatabasePolicyRepository) UpdateDraft(id int64, updatedAt time.Time, config, checksum string) (model.IntelligentRoutingPolicy, error) { ++ return model.UpdateIntelligentRoutingDraft(id, updatedAt, config, checksum) ++} ++ ++func (DatabasePolicyRepository) GetPolicy(id int64) (model.IntelligentRoutingPolicy, error) { ++ return model.GetIntelligentRoutingPolicy(id) ++} ++ ++func (DatabasePolicyRepository) GetPolicyByVersion(version int) (model.IntelligentRoutingPolicy, error) { ++ return model.GetIntelligentRoutingPolicyByVersion(version) ++} ++ ++func (DatabasePolicyRepository) Publish(id int64, administratorID int, note string) (model.IntelligentRoutingPolicy, error) { ++ return model.PublishIntelligentRoutingPolicy(id, administratorID, note) ++} ++ ++func (DatabasePolicyRepository) Rollback(version int, administratorID int, note string) (model.IntelligentRoutingPolicy, error) { ++ return model.RollbackIntelligentRoutingPolicy(version, administratorID, note) ++} ++ ++func (DatabasePolicyRepository) GetRollout() (model.IntelligentRoutingRollout, error) { ++ return model.GetIntelligentRoutingRollout() ++} ++ ++func (DatabasePolicyRepository) UpdateRollout(revision int64, rollout model.IntelligentRoutingRollout) (model.IntelligentRoutingRollout, error) { ++ return model.UpdateIntelligentRoutingRollout(revision, rollout) ++} ++ ++type RuntimeRollout struct { ++ Exists bool ++ Revision int64 ++ PolicyVersion int ++ Enabled bool ++ Mode string ++ TrafficPercent int ++ UserGroups []string ++ TokenGroups []string ++} ++ ++type RuntimePolicySnapshot struct { ++ DeploymentSalt string ++ PolicyID int64 ++ Checksum string ++ Config routingsetting.Config ++ Rollout RuntimeRollout ++} ++ ++type PolicyControl struct { ++ repository PolicyRepository ++ salt string ++ snapshot atomic.Pointer[RuntimePolicySnapshot] ++} ++ ++var DefaultPolicyControl = NewPolicyControl( ++ DatabasePolicyRepository{}, ++ common.GetEnvOrDefaultString("INTELLIGENT_ROUTING_DEPLOYMENT_SALT", "intelligent-routing"), ++) ++ ++func NewPolicyControl(repository PolicyRepository, deploymentSalt string) *PolicyControl { ++ control := &PolicyControl{repository: repository, salt: deploymentSalt} ++ control.snapshot.Store(&RuntimePolicySnapshot{DeploymentSalt: deploymentSalt}) ++ return control ++} ++ ++func (control *PolicyControl) CreateDraft(_ context.Context, raw string, administratorID int) (model.IntelligentRoutingPolicy, []ValidationIssue, error) { ++ validated, issues := ValidatePolicyDocument(raw) ++ if len(issues) > 0 { ++ return model.IntelligentRoutingPolicy{}, issues, nil ++ } ++ policy, err := control.repository.CreateDraft(model.IntelligentRoutingPolicy{ ++ Config: validated.JSON, Checksum: validated.Checksum, CreatedBy: administratorID, ++ }) ++ return policy, nil, err ++} ++ ++func (control *PolicyControl) UpdateDraft(_ context.Context, id int64, updatedAt time.Time, raw string) (model.IntelligentRoutingPolicy, []ValidationIssue, error) { ++ validated, issues := ValidatePolicyDocument(raw) ++ if len(issues) > 0 { ++ return model.IntelligentRoutingPolicy{}, issues, nil ++ } ++ policy, err := control.repository.UpdateDraft(id, updatedAt, validated.JSON, validated.Checksum) ++ return policy, nil, err ++} ++ ++func (control *PolicyControl) Publish(ctx context.Context, id int64, administratorID int, changeNote string) (model.IntelligentRoutingPolicy, []ValidationIssue, error) { ++ if strings.TrimSpace(changeNote) == "" { ++ return model.IntelligentRoutingPolicy{}, []ValidationIssue{{Code: "change_note.required", Field: "change_note", Message: "Change note is required"}}, nil ++ } ++ policy, err := control.repository.GetPolicy(id) ++ if err != nil { ++ return model.IntelligentRoutingPolicy{}, nil, err ++ } ++ validated, issues := ValidatePolicyDocument(policy.Config) ++ if len(issues) > 0 { ++ return model.IntelligentRoutingPolicy{}, issues, nil ++ } ++ if validated.Checksum != policy.Checksum { ++ return model.IntelligentRoutingPolicy{}, []ValidationIssue{{Code: "policy.checksum_mismatch", Field: "policy", Message: "Policy checksum does not match its content"}}, nil ++ } ++ published, err := control.repository.Publish(id, administratorID, strings.TrimSpace(changeNote)) ++ if err != nil { ++ return model.IntelligentRoutingPolicy{}, nil, err ++ } ++ if refreshErr := control.RefreshSnapshot(ctx); refreshErr != nil && !errors.Is(refreshErr, model.ErrIntelligentRoutingRolloutNotFound) { ++ return published, nil, refreshErr ++ } ++ return published, nil, nil ++} ++ ++func (control *PolicyControl) Rollback(ctx context.Context, version, administratorID int, changeNote string) (model.IntelligentRoutingPolicy, []ValidationIssue, error) { ++ if strings.TrimSpace(changeNote) == "" { ++ return model.IntelligentRoutingPolicy{}, []ValidationIssue{{Code: "change_note.required", Field: "change_note", Message: "Change note is required"}}, nil ++ } ++ rolledBack, err := control.repository.Rollback(version, administratorID, strings.TrimSpace(changeNote)) ++ if err != nil { ++ return model.IntelligentRoutingPolicy{}, nil, err ++ } ++ if refreshErr := control.RefreshSnapshot(ctx); refreshErr != nil && !errors.Is(refreshErr, model.ErrIntelligentRoutingRolloutNotFound) { ++ return rolledBack, nil, refreshErr ++ } ++ return rolledBack, nil, nil ++} ++ ++func (control *PolicyControl) UpdateRollout(ctx context.Context, revision int64, rollout model.IntelligentRoutingRollout) (model.IntelligentRoutingRollout, []ValidationIssue, error) { ++ issues := validateRollout(rollout) ++ if len(issues) > 0 { ++ return model.IntelligentRoutingRollout{}, issues, nil ++ } ++ if rollout.PolicyVersion > 0 { ++ policy, err := control.repository.GetPolicyByVersion(rollout.PolicyVersion) ++ if err != nil { ++ return model.IntelligentRoutingRollout{}, nil, err ++ } ++ if policy.Status != model.IntelligentRoutingPolicyActive && rollout.Enabled { ++ return model.IntelligentRoutingRollout{}, []ValidationIssue{{Code: "policy_version.not_active", Field: "policy_version", Message: "Enabled rollout requires the active policy"}}, nil ++ } ++ } ++ updated, err := control.repository.UpdateRollout(revision, rollout) ++ if err != nil { ++ return model.IntelligentRoutingRollout{}, nil, err ++ } ++ if err := control.RefreshSnapshot(ctx); err != nil { ++ return updated, nil, err ++ } ++ return updated, nil, nil ++} ++ ++func validateRollout(rollout model.IntelligentRoutingRollout) []ValidationIssue { ++ if rollout.Mode != model.IntelligentRoutingModeShadow && rollout.Mode != model.IntelligentRoutingModeLive { ++ return []ValidationIssue{{Code: "mode.invalid", Field: "mode", Message: "Rollout mode must be shadow or live"}} ++ } ++ if rollout.TrafficPercent < 0 || rollout.TrafficPercent > 100 { ++ return []ValidationIssue{{Code: "traffic_percent.out_of_range", Field: "traffic_percent", Message: "Traffic percentage must be between 0 and 100"}} ++ } ++ if rollout.Enabled && rollout.PolicyVersion < 1 { ++ return []ValidationIssue{{Code: "policy_version.required", Field: "policy_version", Message: "Enabled rollout requires a policy version"}} ++ } ++ return nil ++} ++ ++func (control *PolicyControl) RefreshSnapshot(_ context.Context) error { ++ rollout, err := control.repository.GetRollout() ++ if errors.Is(err, model.ErrIntelligentRoutingRolloutNotFound) { ++ control.snapshot.Store(&RuntimePolicySnapshot{DeploymentSalt: control.salt}) ++ return nil ++ } ++ if err != nil { ++ return err ++ } ++ snapshot := RuntimePolicySnapshot{DeploymentSalt: control.salt, Rollout: RuntimeRollout{ ++ Exists: true, Revision: rollout.Revision, PolicyVersion: rollout.PolicyVersion, Enabled: rollout.Enabled, ++ Mode: rollout.Mode, TrafficPercent: rollout.TrafficPercent, ++ }} ++ if err := common.UnmarshalJsonStr(rollout.UserGroups, &snapshot.Rollout.UserGroups); err != nil && strings.TrimSpace(rollout.UserGroups) != "" { ++ return err ++ } ++ if err := common.UnmarshalJsonStr(rollout.TokenGroups, &snapshot.Rollout.TokenGroups); err != nil && strings.TrimSpace(rollout.TokenGroups) != "" { ++ return err ++ } ++ if rollout.PolicyVersion > 0 { ++ policy, err := control.repository.GetPolicyByVersion(rollout.PolicyVersion) ++ if err != nil { ++ return err ++ } ++ validated, issues := ValidatePolicyDocument(policy.Config) ++ if len(issues) > 0 || validated.Checksum != policy.Checksum { ++ return errors.New("stored intelligent routing policy failed validation") ++ } ++ snapshot.PolicyID = policy.Id ++ snapshot.Checksum = policy.Checksum ++ snapshot.Config = validated.Config ++ } ++ control.snapshot.Store(&snapshot) ++ return nil ++} ++ ++func (control *PolicyControl) Snapshot() RuntimePolicySnapshot { ++ current := control.snapshot.Load() ++ if current == nil { ++ return RuntimePolicySnapshot{DeploymentSalt: control.salt} ++ } ++ copy := *current ++ copy.Rollout.UserGroups = append([]string(nil), current.Rollout.UserGroups...) ++ copy.Rollout.TokenGroups = append([]string(nil), current.Rollout.TokenGroups...) ++ copy.Config = cloneRoutingConfig(current.Config) ++ return copy ++} ++ ++func cloneRoutingConfig(input routingsetting.Config) routingsetting.Config { ++ input.Models = append([]routingsetting.ModelPolicy(nil), input.Models...) ++ for index := range input.Models { ++ input.Models[index].Capabilities = append([]string(nil), input.Models[index].Capabilities...) ++ } ++ thresholds := make(map[routingsetting.TaskType]float64, len(input.QualityThresholds)) ++ for task, value := range input.QualityThresholds { ++ thresholds[task] = value ++ } ++ input.QualityThresholds = thresholds ++ return input ++} +diff --git a/service/intelligent_routing/policy_control_test.go b/service/intelligent_routing/policy_control_test.go +new file mode 100644 +index 00000000..c549c1eb +--- /dev/null ++++ b/service/intelligent_routing/policy_control_test.go +@@ -0,0 +1,126 @@ ++package intelligent_routing ++ ++import ( ++ "context" ++ "errors" ++ "testing" ++ "time" ++ ++ "github.com/QuantumNous/new-api/model" ++ "github.com/stretchr/testify/assert" ++ "github.com/stretchr/testify/require" ++) ++ ++type policyRepositoryFixture struct { ++ policies map[int64]model.IntelligentRoutingPolicy ++ rollout model.IntelligentRoutingRollout ++ createCalls int ++ err error ++ nextID int64 ++} ++ ++func (repo *policyRepositoryFixture) CreateDraft(policy model.IntelligentRoutingPolicy) (model.IntelligentRoutingPolicy, error) { ++ repo.createCalls++ ++ repo.nextID++ ++ policy.Id = repo.nextID ++ policy.Status = model.IntelligentRoutingPolicyDraft ++ policy.UpdatedAt = time.Now() ++ if repo.policies == nil { ++ repo.policies = make(map[int64]model.IntelligentRoutingPolicy) ++ } ++ repo.policies[policy.Id] = policy ++ return policy, repo.err ++} ++ ++func (repo *policyRepositoryFixture) UpdateDraft(id int64, updatedAt time.Time, config, checksum string) (model.IntelligentRoutingPolicy, error) { ++ policy := repo.policies[id] ++ policy.Config, policy.Checksum = config, checksum ++ repo.policies[id] = policy ++ return policy, repo.err ++} ++ ++func (repo *policyRepositoryFixture) GetPolicy(id int64) (model.IntelligentRoutingPolicy, error) { ++ policy, ok := repo.policies[id] ++ if !ok { ++ return policy, model.ErrIntelligentRoutingPolicyNotFound ++ } ++ return policy, repo.err ++} ++ ++func (repo *policyRepositoryFixture) GetPolicyByVersion(version int) (model.IntelligentRoutingPolicy, error) { ++ for _, policy := range repo.policies { ++ if policy.Version == version { ++ return policy, repo.err ++ } ++ } ++ return model.IntelligentRoutingPolicy{}, model.ErrIntelligentRoutingPolicyNotFound ++} ++ ++func (repo *policyRepositoryFixture) Publish(id int64, administratorID int, note string) (model.IntelligentRoutingPolicy, error) { ++ policy := repo.policies[id] ++ policy.Version, policy.Status, policy.ChangeNote = 1, model.IntelligentRoutingPolicyActive, note ++ repo.policies[id] = policy ++ return policy, repo.err ++} ++ ++func (repo *policyRepositoryFixture) Rollback(version int, administratorID int, note string) (model.IntelligentRoutingPolicy, error) { ++ policy, err := repo.GetPolicyByVersion(version) ++ if err != nil { ++ return policy, err ++ } ++ policy.Version, policy.SourceVersion, policy.ChangeNote = version+1, version, note ++ repo.nextID++ ++ policy.Id = repo.nextID ++ repo.policies[policy.Id] = policy ++ return policy, repo.err ++} ++ ++func (repo *policyRepositoryFixture) GetRollout() (model.IntelligentRoutingRollout, error) { ++ if repo.err != nil { ++ return model.IntelligentRoutingRollout{}, repo.err ++ } ++ if repo.rollout.Id == 0 { ++ return model.IntelligentRoutingRollout{}, model.ErrIntelligentRoutingRolloutNotFound ++ } ++ return repo.rollout, nil ++} ++ ++func (repo *policyRepositoryFixture) UpdateRollout(revision int64, rollout model.IntelligentRoutingRollout) (model.IntelligentRoutingRollout, error) { ++ if repo.err != nil { ++ return model.IntelligentRoutingRollout{}, repo.err ++ } ++ rollout.Id, rollout.Revision = 1, revision+1 ++ repo.rollout = rollout ++ return rollout, nil ++} ++ ++func TestPolicyControlRejectsInvalidDraftBeforeRepositoryWrite(t *testing.T) { ++ repo := &policyRepositoryFixture{} ++ control := NewPolicyControl(repo, "deployment-salt") ++ ++ _, issues, err := control.CreateDraft(context.Background(), `{"max_attempts":99}`, 7) ++ require.NoError(t, err) ++ require.NotEmpty(t, issues) ++ assert.Zero(t, repo.createCalls) ++} ++ ++func TestPolicyControlRefreshRetainsLastValidSnapshot(t *testing.T) { ++ repo := &policyRepositoryFixture{} ++ control := NewPolicyControl(repo, "deployment-salt") ++ draft, issues, err := control.CreateDraft(context.Background(), `{"models":[{"model":"cheap","tier":1,"context_limit":4096}]}`, 7) ++ require.NoError(t, err) ++ require.Empty(t, issues) ++ policy := repo.policies[draft.Id] ++ policy.Version, policy.Status = 1, model.IntelligentRoutingPolicyActive ++ repo.policies[draft.Id] = policy ++ repo.rollout = model.IntelligentRoutingRollout{Id: 1, Revision: 2, PolicyVersion: 1, Enabled: true, Mode: model.IntelligentRoutingModeShadow, TrafficPercent: 100, UserGroups: `[]`, TokenGroups: `[]`} ++ ++ require.NoError(t, control.RefreshSnapshot(context.Background())) ++ before := control.Snapshot() ++ assert.True(t, before.Rollout.Enabled) ++ ++ repo.err = errors.New("database unavailable") ++ assert.Error(t, control.RefreshSnapshot(context.Background())) ++ after := control.Snapshot() ++ assert.Equal(t, before.Rollout.Revision, after.Rollout.Revision) ++} +diff --git a/service/intelligent_routing/policy_document.go b/service/intelligent_routing/policy_document.go +new file mode 100644 +index 00000000..d4d65354 +--- /dev/null ++++ b/service/intelligent_routing/policy_document.go +@@ -0,0 +1,78 @@ ++package intelligent_routing ++ ++import ( ++ "crypto/sha256" ++ "fmt" ++ "sort" ++ ++ "github.com/QuantumNous/new-api/common" ++ routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" ++) ++ ++type ValidationIssue struct { ++ Code string `json:"code"` ++ Field string `json:"field"` ++ Message string `json:"message"` ++} ++ ++type ValidatedPolicy struct { ++ Config routingsetting.Config ++ JSON string ++ Checksum string ++} ++ ++func ValidatePolicyDocument(raw string) (ValidatedPolicy, []ValidationIssue) { ++ if len(raw) > routingsetting.MaxPolicyDocumentBytes { ++ return ValidatedPolicy{}, []ValidationIssue{{Code: "policy.too_large", Field: "policy", Message: "Policy document is too large"}} ++ } ++ var input routingsetting.Config ++ if err := common.UnmarshalJsonStr(raw, &input); err != nil { ++ return ValidatedPolicy{}, []ValidationIssue{{Code: "policy.invalid_json", Field: "policy", Message: "Policy document is not valid JSON"}} ++ } ++ if input.MaxAttempts > routingsetting.MaxAttempts || input.MaxAttempts < 0 { ++ return ValidatedPolicy{}, []ValidationIssue{{Code: "max_attempts.out_of_range", Field: "max_attempts", Message: "Maximum attempts is out of range"}} ++ } ++ if input.MaxEndpointsPerModel > routingsetting.MaxEndpointsPerModel || input.MaxEndpointsPerModel < 0 { ++ return ValidatedPolicy{}, []ValidationIssue{{Code: "max_endpoints_per_model.out_of_range", Field: "max_endpoints_per_model", Message: "Maximum endpoints per model is out of range"}} ++ } ++ allowedCapabilities := map[string]struct{}{ ++ string(CapabilityTools): {}, string(CapabilityJSONSchema): {}, string(CapabilityVision): {}, string(CapabilityAudio): {}, ++ } ++ for modelIndex, policy := range input.Models { ++ for capabilityIndex, capability := range policy.Capabilities { ++ if _, ok := allowedCapabilities[capability]; !ok { ++ return ValidatedPolicy{}, []ValidationIssue{{ ++ Code: "models.capability.unknown", Field: fmt.Sprintf("models[%d].capabilities[%d]", modelIndex, capabilityIndex), ++ Message: "Model capability is unknown", ++ }} ++ } ++ } ++ } ++ ++ normalized, err := routingsetting.Normalize(input) ++ if err != nil { ++ return ValidatedPolicy{}, []ValidationIssue{{Code: "policy.invalid", Field: "policy", Message: "Policy document contains invalid values"}} ++ } ++ canonical, err := CanonicalPolicyJSON(normalized) ++ if err != nil { ++ return ValidatedPolicy{}, []ValidationIssue{{Code: "policy.canonicalization_failed", Field: "policy", Message: "Policy document could not be canonicalized"}} ++ } ++ sum := sha256.Sum256([]byte(canonical)) ++ return ValidatedPolicy{Config: normalized, JSON: canonical, Checksum: fmt.Sprintf("%x", sum)}, nil ++} ++ ++func CanonicalPolicyJSON(config routingsetting.Config) (string, error) { ++ config.Models = append([]routingsetting.ModelPolicy(nil), config.Models...) ++ for index := range config.Models { ++ config.Models[index].Capabilities = append([]string(nil), config.Models[index].Capabilities...) ++ sort.Strings(config.Models[index].Capabilities) ++ } ++ sort.Slice(config.Models, func(i, j int) bool { ++ return config.Models[i].Model < config.Models[j].Model ++ }) ++ data, err := common.Marshal(config) ++ if err != nil { ++ return "", err ++ } ++ return string(data), nil ++} +diff --git a/service/intelligent_routing/policy_document_test.go b/service/intelligent_routing/policy_document_test.go +new file mode 100644 +index 00000000..927c11c0 +--- /dev/null ++++ b/service/intelligent_routing/policy_document_test.go +@@ -0,0 +1,43 @@ ++package intelligent_routing ++ ++import ( ++ "strings" ++ "testing" ++ ++ routingsetting "github.com/QuantumNous/new-api/setting/intelligent_routing_setting" ++ "github.com/stretchr/testify/assert" ++ "github.com/stretchr/testify/require" ++) ++ ++func TestValidatePolicyDocumentReturnsStructuredIssues(t *testing.T) { ++ tests := []struct { ++ name string ++ raw string ++ code string ++ field string ++ }{ ++ {name: "malformed", raw: `{`, code: "policy.invalid_json", field: "policy"}, ++ {name: "too large", raw: `{"padding":"` + strings.Repeat("x", routingsetting.MaxPolicyDocumentBytes) + `"}`, code: "policy.too_large", field: "policy"}, ++ {name: "attempts", raw: `{"max_attempts":99}`, code: "max_attempts.out_of_range", field: "max_attempts"}, ++ {name: "capability", raw: `{"models":[{"model":"cheap","tier":1,"context_limit":4096,"capabilities":["telepathy"]}]}`, code: "models.capability.unknown", field: "models[0].capabilities[0]"}, ++ } ++ ++ for _, test := range tests { ++ t.Run(test.name, func(t *testing.T) { ++ _, issues := ValidatePolicyDocument(test.raw) ++ require.NotEmpty(t, issues) ++ assert.Equal(t, test.code, issues[0].Code) ++ assert.Equal(t, test.field, issues[0].Field) ++ }) ++ } ++} ++ ++func TestCanonicalPolicyJSONProducesStableChecksum(t *testing.T) { ++ first, issues := ValidatePolicyDocument(`{"models":[{"model":"b","tier":1,"context_limit":4096,"capabilities":["tools","json_schema"]},{"model":"a","tier":0,"context_limit":2048}]}`) ++ require.Empty(t, issues) ++ second, issues := ValidatePolicyDocument(`{"models":[{"model":"a","tier":0,"context_limit":2048},{"model":"b","tier":1,"context_limit":4096,"capabilities":["json_schema","tools"]}]}`) ++ require.Empty(t, issues) ++ ++ assert.Equal(t, first.Checksum, second.Checksum) ++ assert.Equal(t, first.JSON, second.JSON) ++} +diff --git a/service/intelligent_routing/policy_refresh.go b/service/intelligent_routing/policy_refresh.go +new file mode 100644 +index 00000000..2f2847ee +--- /dev/null ++++ b/service/intelligent_routing/policy_refresh.go +@@ -0,0 +1,38 @@ ++package intelligent_routing ++ ++import ( ++ "context" ++ "time" ++ ++ "github.com/QuantumNous/new-api/common" ++) ++ ++func StartPolicyRefresh(ctx context.Context, control *PolicyControl, interval time.Duration) { ++ if interval <= 0 { ++ interval = time.Minute ++ } ++ ticker := time.NewTicker(interval) ++ go func() { ++ defer ticker.Stop() ++ runPolicyRefresh(ctx, control, ticker.C) ++ }() ++} ++ ++func runPolicyRefresh(ctx context.Context, control *PolicyControl, triggers <-chan time.Time) { ++ failed := false ++ for { ++ select { ++ case <-ctx.Done(): ++ return ++ case <-triggers: ++ if err := control.RefreshSnapshot(ctx); err != nil { ++ if !failed { ++ common.SysError("failed to refresh intelligent routing policy snapshot: " + err.Error()) ++ } ++ failed = true ++ continue ++ } ++ failed = false ++ } ++ } ++} +diff --git a/service/intelligent_routing/policy_refresh_test.go b/service/intelligent_routing/policy_refresh_test.go +new file mode 100644 +index 00000000..5a747f19 +--- /dev/null ++++ b/service/intelligent_routing/policy_refresh_test.go +@@ -0,0 +1,30 @@ ++package intelligent_routing ++ ++import ( ++ "context" ++ "testing" ++ "time" ++ ++ "github.com/stretchr/testify/assert" ++) ++ ++func TestPolicyRefreshStopsAfterCancellation(t *testing.T) { ++ repo := &policyRepositoryFixture{} ++ control := NewPolicyControl(repo, "salt") ++ triggers := make(chan time.Time, 2) ++ ctx, cancel := context.WithCancel(context.Background()) ++ done := make(chan struct{}) ++ go func() { ++ runPolicyRefresh(ctx, control, triggers) ++ close(done) ++ }() ++ ++ triggers <- time.Now() ++ cancel() ++ select { ++ case <-done: ++ case <-time.After(time.Second): ++ t.Fatal("refresh loop did not stop after cancellation") ++ } ++ assert.False(t, control.Snapshot().Rollout.Enabled) ++} +diff --git a/service/intelligent_routing/rollout.go b/service/intelligent_routing/rollout.go +new file mode 100644 +index 00000000..34fe0c6f +--- /dev/null ++++ b/service/intelligent_routing/rollout.go +@@ -0,0 +1,54 @@ ++package intelligent_routing ++ ++import ( ++ "crypto/hmac" ++ "crypto/sha256" ++ "encoding/binary" ++ "fmt" ++) ++ ++type RolloutSubject struct { ++ AccountID int ++ TokenID int ++ UserGroup string ++ TokenGroup string ++} ++ ++type RolloutDecision struct { ++ Selected bool ++ Bucket int ++ Mode string ++ PolicyVersion int ++ Revision int64 ++} ++ ++func ResolveRollout(snapshot RuntimePolicySnapshot, subject RolloutSubject) RolloutDecision { ++ decision := RolloutDecision{ ++ Mode: snapshot.Rollout.Mode, PolicyVersion: snapshot.Rollout.PolicyVersion, Revision: snapshot.Rollout.Revision, ++ } ++ if !snapshot.Rollout.Exists || !snapshot.Rollout.Enabled || snapshot.Rollout.TrafficPercent <= 0 { ++ return decision ++ } ++ if !rolloutGroupMatches(snapshot.Rollout.UserGroups, subject.UserGroup) || ++ !rolloutGroupMatches(snapshot.Rollout.TokenGroups, subject.TokenGroup) { ++ return decision ++ } ++ mac := hmac.New(sha256.New, []byte(snapshot.DeploymentSalt)) ++ _, _ = fmt.Fprintf(mac, "%d/%d/%d", snapshot.Rollout.PolicyVersion, subject.AccountID, subject.TokenID) ++ digest := mac.Sum(nil) ++ decision.Bucket = int(binary.BigEndian.Uint64(digest[:8]) % 100) ++ decision.Selected = decision.Bucket < snapshot.Rollout.TrafficPercent ++ return decision ++} ++ ++func rolloutGroupMatches(allowed []string, actual string) bool { ++ if len(allowed) == 0 { ++ return true ++ } ++ for _, candidate := range allowed { ++ if candidate == actual { ++ return true ++ } ++ } ++ return false ++} +diff --git a/service/intelligent_routing/rollout_test.go b/service/intelligent_routing/rollout_test.go +new file mode 100644 +index 00000000..fea1c271 +--- /dev/null ++++ b/service/intelligent_routing/rollout_test.go +@@ -0,0 +1,39 @@ ++package intelligent_routing ++ ++import ( ++ "testing" ++ ++ "github.com/stretchr/testify/assert" ++) ++ ++func TestResolveRolloutUsesStableSubjectBucket(t *testing.T) { ++ snapshot := RuntimePolicySnapshot{DeploymentSalt: "salt", Rollout: RuntimeRollout{ ++ Exists: true, Revision: 3, PolicyVersion: 2, Enabled: true, Mode: "live", TrafficPercent: 100, ++ UserGroups: []string{"default"}, TokenGroups: []string{"auto"}, ++ }} ++ subject := RolloutSubject{AccountID: 42, TokenID: 9, UserGroup: "default", TokenGroup: "auto"} ++ ++ first := ResolveRollout(snapshot, subject) ++ second := ResolveRollout(snapshot, subject) ++ ++ assert.True(t, first.Selected) ++ assert.Equal(t, first.Bucket, second.Bucket) ++ assert.Equal(t, "live", first.Mode) ++} ++ ++func TestResolveRolloutRejectsDisabledExcludedAndZeroPercent(t *testing.T) { ++ base := RuntimePolicySnapshot{DeploymentSalt: "salt", Rollout: RuntimeRollout{ ++ Exists: true, PolicyVersion: 1, Enabled: true, Mode: "shadow", TrafficPercent: 100, ++ UserGroups: []string{"allowed"}, TokenGroups: []string{"auto"}, ++ }} ++ subject := RolloutSubject{AccountID: 1, TokenID: 2, UserGroup: "other", TokenGroup: "auto"} ++ assert.False(t, ResolveRollout(base, subject).Selected) ++ ++ base.Rollout.UserGroups = nil ++ base.Rollout.Enabled = false ++ assert.False(t, ResolveRollout(base, subject).Selected) ++ ++ base.Rollout.Enabled = true ++ base.Rollout.TrafficPercent = 0 ++ assert.False(t, ResolveRollout(base, subject).Selected) ++} +diff --git a/service/log_info_generate.go b/service/log_info_generate.go +index e6be8579..1296b957 100644 +--- a/service/log_info_generate.go ++++ b/service/log_info_generate.go +@@ -133,6 +133,12 @@ func appendIntelligentRoutingAdminInfo(relayInfo *relaycommon.RelayInfo, adminIn + "execution_model": relayInfo.GetExecutionModelName(), + "attempt_index": relayInfo.IntelligentRouteAttempt, + } ++ if relayInfo.IntelligentRouteRolloutRevision > 0 { ++ audit["rollout_revision"] = relayInfo.IntelligentRouteRolloutRevision ++ audit["rollout_bucket"] = relayInfo.IntelligentRouteRolloutBucket ++ audit["rollout_mode"] = relayInfo.IntelligentRouteRolloutMode ++ audit["policy_version"] = relayInfo.IntelligentRoutePolicyVersion ++ } + if relayInfo.IntelligentRouteError != "" { + audit["error"] = relayInfo.IntelligentRouteError + } +diff --git a/setting/intelligent_routing_setting/config.go b/setting/intelligent_routing_setting/config.go +index f7f1064c..e777549a 100644 +--- a/setting/intelligent_routing_setting/config.go ++++ b/setting/intelligent_routing_setting/config.go +@@ -20,6 +20,15 @@ const ( + TaskReasoning TaskType = "reasoning" + TaskJSON TaskType = "json_schema" + TaskTool TaskType = "tool" ++ ++ MaxPolicyDocumentBytes = 1 << 20 ++ MaxPolicyModels = 512 ++ MaxAttempts = 8 ++ MaxEndpointsPerModel = 4 ++ MaxExecutionBudget = 2 * time.Minute ++ MaxContextLimit = 10_000_000 ++ MaxModelPrice = 1_000_000 ++ MaxCostMultiplier = 100 + ) + + type ModelPolicy struct { +@@ -73,7 +82,11 @@ func Normalize(input Config) (Config, error) { + if input.MaxCostMultiplier == 0 { + input.MaxCostMultiplier = 2.5 + } +- if input.PolicyVersion < 1 || input.MaxAttempts < 1 || input.MaxEndpointsPerModel < 1 || input.NonStreamBudget < 0 || input.StreamFirstByteBudget < 0 || input.MaxCostMultiplier < 1 { ++ if input.PolicyVersion < 1 || input.MaxAttempts < 1 || input.MaxAttempts > MaxAttempts || ++ input.MaxEndpointsPerModel < 1 || input.MaxEndpointsPerModel > MaxEndpointsPerModel || ++ input.NonStreamBudget < 0 || input.NonStreamBudget > MaxExecutionBudget || ++ input.StreamFirstByteBudget < 0 || input.StreamFirstByteBudget > MaxExecutionBudget || ++ input.MaxCostMultiplier < 1 || input.MaxCostMultiplier > MaxCostMultiplier { + return Config{}, errors.New("invalid intelligent routing budget") + } + defaults := map[TaskType]float64{ +@@ -87,9 +100,15 @@ func Normalize(input Config) (Config, error) { + defaults[task] = value + } + input.QualityThresholds = defaults ++ if len(input.Models) > MaxPolicyModels { ++ return Config{}, errors.New("too many intelligent routing model policies") ++ } + seen := make(map[string]struct{}, len(input.Models)) + for _, policy := range input.Models { +- if policy.Model == "" || policy.Tier < 0 || policy.Tier > 3 || policy.InputPrice < 0 || policy.OutputPrice < 0 || policy.ContextLimit < 0 { ++ if policy.Model == "" || policy.Tier < 0 || policy.Tier > 3 || ++ policy.InputPrice < 0 || policy.InputPrice > MaxModelPrice || ++ policy.OutputPrice < 0 || policy.OutputPrice > MaxModelPrice || ++ policy.ContextLimit < 0 || policy.ContextLimit > MaxContextLimit { + return Config{}, fmt.Errorf("invalid model policy for %q", policy.Model) + } + if _, ok := seen[policy.Model]; ok { +diff --git a/setting/intelligent_routing_setting/config_test.go b/setting/intelligent_routing_setting/config_test.go +index 3d4f748c..65e4f684 100644 +--- a/setting/intelligent_routing_setting/config_test.go ++++ b/setting/intelligent_routing_setting/config_test.go +@@ -23,6 +23,11 @@ func TestNormalizeConfigAppliesSafeDefaults(t *testing.T) { + func TestNormalizeConfigRejectsInvalidValues(t *testing.T) { + tests := []Config{ + {MaxAttempts: -1}, ++ {MaxAttempts: MaxAttempts + 1}, ++ {MaxEndpointsPerModel: MaxEndpointsPerModel + 1}, ++ {NonStreamBudget: MaxExecutionBudget + time.Nanosecond}, ++ {StreamFirstByteBudget: MaxExecutionBudget + time.Nanosecond}, ++ {MaxCostMultiplier: MaxCostMultiplier + 0.01}, + {QualityThresholds: map[TaskType]float64{TaskGeneral: 1.1}}, + {Models: []ModelPolicy{{Model: "a", Tier: 4}}}, + {Models: []ModelPolicy{{Model: "a"}, {Model: "a"}}}, diff --git a/verification-intelligent-routing-policy-control/MODIFIED_FILE b/verification-intelligent-routing-policy-control/MODIFIED_FILE new file mode 100644 index 000000000000..e777549afa73 --- /dev/null +++ b/verification-intelligent-routing-policy-control/MODIFIED_FILE @@ -0,0 +1,159 @@ +package intelligent_routing_setting + +import ( + "errors" + "fmt" + "sync/atomic" + "time" + + "github.com/QuantumNous/new-api/setting/config" +) + +type TaskType string + +const ( + TaskTranslation TaskType = "translation" + TaskSummary TaskType = "summary" + TaskGeneral TaskType = "general" + TaskExtraction TaskType = "extraction" + TaskCode TaskType = "code" + TaskReasoning TaskType = "reasoning" + TaskJSON TaskType = "json_schema" + TaskTool TaskType = "tool" + + MaxPolicyDocumentBytes = 1 << 20 + MaxPolicyModels = 512 + MaxAttempts = 8 + MaxEndpointsPerModel = 4 + MaxExecutionBudget = 2 * time.Minute + MaxContextLimit = 10_000_000 + MaxModelPrice = 1_000_000 + MaxCostMultiplier = 100 +) + +type ModelPolicy struct { + Model string `json:"model"` + Tier int `json:"tier"` + InputPrice float64 `json:"input_price"` + OutputPrice float64 `json:"output_price"` + ContextLimit int `json:"context_limit"` + Capabilities []string `json:"capabilities"` +} + +type Config struct { + Enabled bool `json:"enabled"` + ShadowOnly bool `json:"shadow_only"` + PolicyVersion int `json:"policy_version"` + MaxAttempts int `json:"max_attempts"` + MaxEndpointsPerModel int `json:"max_endpoints_per_model"` + NonStreamBudget time.Duration `json:"non_stream_budget"` + StreamFirstByteBudget time.Duration `json:"stream_first_byte_budget"` + MaxCostMultiplier float64 `json:"max_cost_multiplier"` + QualityThresholds map[TaskType]float64 `json:"quality_thresholds"` + Models []ModelPolicy `json:"models"` +} + +var current atomic.Pointer[Config] +var registeredConfig Config + +func init() { + normalized, _ := Normalize(Config{}) + registeredConfig = normalized + current.Store(&normalized) + config.GlobalConfig.Register("intelligent_routing_setting", ®isteredConfig) +} + +func Normalize(input Config) (Config, error) { + if input.PolicyVersion == 0 { + input.PolicyVersion = 1 + } + if input.MaxAttempts == 0 { + input.MaxAttempts = 4 + } + if input.MaxEndpointsPerModel == 0 { + input.MaxEndpointsPerModel = 2 + } + if input.NonStreamBudget == 0 { + input.NonStreamBudget = 30 * time.Second + } + if input.StreamFirstByteBudget == 0 { + input.StreamFirstByteBudget = 12 * time.Second + } + if input.MaxCostMultiplier == 0 { + input.MaxCostMultiplier = 2.5 + } + if input.PolicyVersion < 1 || input.MaxAttempts < 1 || input.MaxAttempts > MaxAttempts || + input.MaxEndpointsPerModel < 1 || input.MaxEndpointsPerModel > MaxEndpointsPerModel || + input.NonStreamBudget < 0 || input.NonStreamBudget > MaxExecutionBudget || + input.StreamFirstByteBudget < 0 || input.StreamFirstByteBudget > MaxExecutionBudget || + input.MaxCostMultiplier < 1 || input.MaxCostMultiplier > MaxCostMultiplier { + return Config{}, errors.New("invalid intelligent routing budget") + } + defaults := map[TaskType]float64{ + TaskTranslation: .88, TaskSummary: .88, TaskGeneral: .90, TaskCode: .93, TaskExtraction: .94, + TaskReasoning: .95, TaskJSON: .97, TaskTool: .98, + } + for task, value := range input.QualityThresholds { + if value < 0 || value > 1 { + return Config{}, fmt.Errorf("quality threshold for %s must be between 0 and 1", task) + } + defaults[task] = value + } + input.QualityThresholds = defaults + if len(input.Models) > MaxPolicyModels { + return Config{}, errors.New("too many intelligent routing model policies") + } + seen := make(map[string]struct{}, len(input.Models)) + for _, policy := range input.Models { + if policy.Model == "" || policy.Tier < 0 || policy.Tier > 3 || + policy.InputPrice < 0 || policy.InputPrice > MaxModelPrice || + policy.OutputPrice < 0 || policy.OutputPrice > MaxModelPrice || + policy.ContextLimit < 0 || policy.ContextLimit > MaxContextLimit { + return Config{}, fmt.Errorf("invalid model policy for %q", policy.Model) + } + if _, ok := seen[policy.Model]; ok { + return Config{}, fmt.Errorf("duplicate model policy %q", policy.Model) + } + seen[policy.Model] = struct{}{} + } + return clone(input), nil +} + +func Update(input Config) error { + normalized, err := Normalize(input) + if err != nil { + return err + } + current.Store(&normalized) + registeredConfig = clone(normalized) + return nil +} + +func UpdateAndSync() error { + return Update(registeredConfig) +} + +func Get() Config { + value := current.Load() + if value == nil { + return Config{} + } + return clone(*value) +} + +func Enabled() bool { + return Get().Enabled +} + +func clone(input Config) Config { + input.Models = append([]ModelPolicy(nil), input.Models...) + for i := range input.Models { + input.Models[i].Capabilities = append([]string(nil), input.Models[i].Capabilities...) + } + thresholds := input.QualityThresholds + input.QualityThresholds = make(map[TaskType]float64, len(thresholds)) + for task, value := range thresholds { + input.QualityThresholds[task] = value + } + return input +} diff --git a/verification-intelligent-routing-policy-control/ORIGINAL_FILE b/verification-intelligent-routing-policy-control/ORIGINAL_FILE new file mode 100644 index 000000000000..a191b669fa66 --- /dev/null +++ b/verification-intelligent-routing-policy-control/ORIGINAL_FILE @@ -0,0 +1 @@ +package intelligent_routing_settingimport ( "errors" "fmt" "sync/atomic" "time" "github.com/QuantumNous/new-api/setting/config")type TaskType stringconst ( TaskTranslation TaskType = "translation" TaskSummary TaskType = "summary" TaskGeneral TaskType = "general" TaskExtraction TaskType = "extraction" TaskCode TaskType = "code" TaskReasoning TaskType = "reasoning" TaskJSON TaskType = "json_schema" TaskTool TaskType = "tool")type ModelPolicy struct { Model string `json:"model"` Tier int `json:"tier"` InputPrice float64 `json:"input_price"` OutputPrice float64 `json:"output_price"` ContextLimit int `json:"context_limit"` Capabilities []string `json:"capabilities"`}type Config struct { Enabled bool `json:"enabled"` ShadowOnly bool `json:"shadow_only"` PolicyVersion int `json:"policy_version"` MaxAttempts int `json:"max_attempts"` MaxEndpointsPerModel int `json:"max_endpoints_per_model"` NonStreamBudget time.Duration `json:"non_stream_budget"` StreamFirstByteBudget time.Duration `json:"stream_first_byte_budget"` MaxCostMultiplier float64 `json:"max_cost_multiplier"` QualityThresholds map[TaskType]float64 `json:"quality_thresholds"` Models []ModelPolicy `json:"models"`}var current atomic.Pointer[Config]var registeredConfig Configfunc init() { normalized, _ := Normalize(Config{}) registeredConfig = normalized current.Store(&normalized) config.GlobalConfig.Register("intelligent_routing_setting", ®isteredConfig)}func Normalize(input Config) (Config, error) { if input.PolicyVersion == 0 { input.PolicyVersion = 1 } if input.MaxAttempts == 0 { input.MaxAttempts = 4 } if input.MaxEndpointsPerModel == 0 { input.MaxEndpointsPerModel = 2 } if input.NonStreamBudget == 0 { input.NonStreamBudget = 30 * time.Second } if input.StreamFirstByteBudget == 0 { input.StreamFirstByteBudget = 12 * time.Second } if input.MaxCostMultiplier == 0 { input.MaxCostMultiplier = 2.5 } if input.PolicyVersion < 1 || input.MaxAttempts < 1 || input.MaxEndpointsPerModel < 1 || input.NonStreamBudget < 0 || input.StreamFirstByteBudget < 0 || input.MaxCostMultiplier < 1 { return Config{}, errors.New("invalid intelligent routing budget") } defaults := map[TaskType]float64{ TaskTranslation: .88, TaskSummary: .88, TaskGeneral: .90, TaskCode: .93, TaskExtraction: .94, TaskReasoning: .95, TaskJSON: .97, TaskTool: .98, } for task, value := range input.QualityThresholds { if value < 0 || value > 1 { return Config{}, fmt.Errorf("quality threshold for %s must be between 0 and 1", task) } defaults[task] = value } input.QualityThresholds = defaults seen := make(map[string]struct{}, len(input.Models)) for _, policy := range input.Models { if policy.Model == "" || policy.Tier < 0 || policy.Tier > 3 || policy.InputPrice < 0 || policy.OutputPrice < 0 || policy.ContextLimit < 0 { return Config{}, fmt.Errorf("invalid model policy for %q", policy.Model) } if _, ok := seen[policy.Model]; ok { return Config{}, fmt.Errorf("duplicate model policy %q", policy.Model) } seen[policy.Model] = struct{}{} } return clone(input), nil}func Update(input Config) error { normalized, err := Normalize(input) if err != nil { return err } current.Store(&normalized) registeredConfig = clone(normalized) return nil}func UpdateAndSync() error { return Update(registeredConfig)}func Get() Config { value := current.Load() if value == nil { return Config{} } return clone(*value)}func Enabled() bool { return Get().Enabled}func clone(input Config) Config { input.Models = append([]ModelPolicy(nil), input.Models...) for i := range input.Models { input.Models[i].Capabilities = append([]string(nil), input.Models[i].Capabilities...) } thresholds := input.QualityThresholds input.QualityThresholds = make(map[TaskType]float64, len(thresholds)) for task, value := range thresholds { input.QualityThresholds[task] = value } return input} \ No newline at end of file diff --git a/verification-intelligent-routing-policy-control/ROLLBACK.sh b/verification-intelligent-routing-policy-control/ROLLBACK.sh new file mode 100755 index 000000000000..57fd5b5ecbe5 --- /dev/null +++ b/verification-intelligent-routing-policy-control/ROLLBACK.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +DIR="$(cd "$(dirname "$0")" && pwd)" +cp "$DIR/ORIGINAL_FILE" "$DIR/ROLLBACK_TEST_COPY" +sha256sum "$DIR/ROLLBACK_TEST_COPY" | awk '{print toupper($1)}' \ No newline at end of file diff --git a/verification-intelligent-routing-policy-control/ROLLBACK_TEST_COPY b/verification-intelligent-routing-policy-control/ROLLBACK_TEST_COPY new file mode 100644 index 000000000000..a191b669fa66 --- /dev/null +++ b/verification-intelligent-routing-policy-control/ROLLBACK_TEST_COPY @@ -0,0 +1 @@ +package intelligent_routing_settingimport ( "errors" "fmt" "sync/atomic" "time" "github.com/QuantumNous/new-api/setting/config")type TaskType stringconst ( TaskTranslation TaskType = "translation" TaskSummary TaskType = "summary" TaskGeneral TaskType = "general" TaskExtraction TaskType = "extraction" TaskCode TaskType = "code" TaskReasoning TaskType = "reasoning" TaskJSON TaskType = "json_schema" TaskTool TaskType = "tool")type ModelPolicy struct { Model string `json:"model"` Tier int `json:"tier"` InputPrice float64 `json:"input_price"` OutputPrice float64 `json:"output_price"` ContextLimit int `json:"context_limit"` Capabilities []string `json:"capabilities"`}type Config struct { Enabled bool `json:"enabled"` ShadowOnly bool `json:"shadow_only"` PolicyVersion int `json:"policy_version"` MaxAttempts int `json:"max_attempts"` MaxEndpointsPerModel int `json:"max_endpoints_per_model"` NonStreamBudget time.Duration `json:"non_stream_budget"` StreamFirstByteBudget time.Duration `json:"stream_first_byte_budget"` MaxCostMultiplier float64 `json:"max_cost_multiplier"` QualityThresholds map[TaskType]float64 `json:"quality_thresholds"` Models []ModelPolicy `json:"models"`}var current atomic.Pointer[Config]var registeredConfig Configfunc init() { normalized, _ := Normalize(Config{}) registeredConfig = normalized current.Store(&normalized) config.GlobalConfig.Register("intelligent_routing_setting", ®isteredConfig)}func Normalize(input Config) (Config, error) { if input.PolicyVersion == 0 { input.PolicyVersion = 1 } if input.MaxAttempts == 0 { input.MaxAttempts = 4 } if input.MaxEndpointsPerModel == 0 { input.MaxEndpointsPerModel = 2 } if input.NonStreamBudget == 0 { input.NonStreamBudget = 30 * time.Second } if input.StreamFirstByteBudget == 0 { input.StreamFirstByteBudget = 12 * time.Second } if input.MaxCostMultiplier == 0 { input.MaxCostMultiplier = 2.5 } if input.PolicyVersion < 1 || input.MaxAttempts < 1 || input.MaxEndpointsPerModel < 1 || input.NonStreamBudget < 0 || input.StreamFirstByteBudget < 0 || input.MaxCostMultiplier < 1 { return Config{}, errors.New("invalid intelligent routing budget") } defaults := map[TaskType]float64{ TaskTranslation: .88, TaskSummary: .88, TaskGeneral: .90, TaskCode: .93, TaskExtraction: .94, TaskReasoning: .95, TaskJSON: .97, TaskTool: .98, } for task, value := range input.QualityThresholds { if value < 0 || value > 1 { return Config{}, fmt.Errorf("quality threshold for %s must be between 0 and 1", task) } defaults[task] = value } input.QualityThresholds = defaults seen := make(map[string]struct{}, len(input.Models)) for _, policy := range input.Models { if policy.Model == "" || policy.Tier < 0 || policy.Tier > 3 || policy.InputPrice < 0 || policy.OutputPrice < 0 || policy.ContextLimit < 0 { return Config{}, fmt.Errorf("invalid model policy for %q", policy.Model) } if _, ok := seen[policy.Model]; ok { return Config{}, fmt.Errorf("duplicate model policy %q", policy.Model) } seen[policy.Model] = struct{}{} } return clone(input), nil}func Update(input Config) error { normalized, err := Normalize(input) if err != nil { return err } current.Store(&normalized) registeredConfig = clone(normalized) return nil}func UpdateAndSync() error { return Update(registeredConfig)}func Get() Config { value := current.Load() if value == nil { return Config{} } return clone(*value)}func Enabled() bool { return Get().Enabled}func clone(input Config) Config { input.Models = append([]ModelPolicy(nil), input.Models...) for i := range input.Models { input.Models[i].Capabilities = append([]string(nil), input.Models[i].Capabilities...) } thresholds := input.QualityThresholds input.QualityThresholds = make(map[TaskType]float64, len(thresholds)) for task, value := range thresholds { input.QualityThresholds[task] = value } return input} \ No newline at end of file diff --git a/verification-intelligent-routing-policy-control/VERIFICATION.txt b/verification-intelligent-routing-policy-control/VERIFICATION.txt new file mode 100644 index 000000000000..9906bf95089f --- /dev/null +++ b/verification-intelligent-routing-policy-control/VERIFICATION.txt @@ -0,0 +1,34 @@ +CHANGED_BRANCH/FIELD: codex/intelligent-routing-admin; durable administrator policy versions, scoped rollout, validation, publication, rollback, runtime snapshot, request-path activation, and routing audit metadata. +MODIFIED_FILE: C:\Users\liwen\Documents\ChatGPT\中转站\.worktrees\intelligent-routing-admin\verification-intelligent-routing-policy-control\MODIFIED_FILE +DIFF_FILE: C:\Users\liwen\Documents\ChatGPT\中转站\.worktrees\intelligent-routing-admin\verification-intelligent-routing-policy-control\DIFF_FILE +VERIFICATION: C:\Users\liwen\Documents\ChatGPT\中转站\.worktrees\intelligent-routing-admin\verification-intelligent-routing-policy-control\VERIFICATION.txt +ROLLBACK: C:\Users\liwen\Documents\ChatGPT\中转站\.worktrees\intelligent-routing-admin\verification-intelligent-routing-policy-control\ROLLBACK.sh +BASELINE COMMAND: Get-FileHash -Algorithm SHA256 -LiteralPath verification-intelligent-routing-policy-control\ORIGINAL_FILE +BASELINE INPUT: commit faab9bcd setting/intelligent_routing_setting/config.go +BASELINE OUTPUT/RESULT: F8DCB7E1F39E890E7B22886E709FBC9F7E69BF45ED2866E4AF4BB2815BD2B621 +BASELINE EXIT STATUS: 0 +MODIFIED COMMAND: Get-FileHash -Algorithm SHA256 -LiteralPath verification-intelligent-routing-policy-control\MODIFIED_FILE +MODIFIED INPUT: current codex/intelligent-routing-admin setting/intelligent_routing_setting/config.go +MODIFIED OUTPUT/RESULT: D70C0641C71F002BDD6A4003FBC621E57AAE4DA8EA6355C82D2054CAC99E5E4F +MODIFIED EXIT STATUS: 0 +TEST COMMAND: $env:GOCACHE="$PWD\.gocache"; go test ./... -count=1 +TEST INPUT: complete root Go module with embedded web/dist fixture +TEST OUTPUT/RESULT: all root-module packages passed; github.com/QuantumNous/new-api/controller ok 33.550s; github.com/QuantumNous/new-api/model ok 39.264s; github.com/QuantumNous/new-api/router ok 11.529s; github.com/QuantumNous/new-api/service/intelligent_routing ok 9.126s +TEST EXIT STATUS: 0 +VET COMMAND: go vet ./service/intelligent_routing ./controller ./model ./router +VET INPUT: modified intelligent-routing service, controller, model, and router packages +VET OUTPUT/RESULT: no output +VET EXIT STATUS: 0 +RELAYKIT COMMAND: $env:GOWORK='off'; go build ./... +RELAYKIT INPUT: C:\Users\liwen\Documents\ChatGPT\中转站\.worktrees\intelligent-routing-admin\relaykit independent module +RELAYKIT OUTPUT/RESULT: no output +RELAYKIT EXIT STATUS: 0 +DIFF CHECK COMMAND: git diff --check +DIFF CHECK INPUT: tracked source tree +DIFF CHECK OUTPUT/RESULT: no output +DIFF CHECK EXIT STATUS: 0 +ROLLBACK COMMAND: C:\Program Files\Git\bin\bash.exe verification-intelligent-routing-policy-control\ROLLBACK.sh +ROLLBACK INPUT: MODIFIED_FILE copied to ROLLBACK_TEST_COPY, then ORIGINAL_FILE restored onto ROLLBACK_TEST_COPY +ROLLBACK OUTPUT/RESULT: F8DCB7E1F39E890E7B22886E709FBC9F7E69BF45ED2866E4AF4BB2815BD2B621; ROLLBACK_MATCH=True +ROLLBACK EXIT STATUS: 0 +RESTORED BEHAVIOR/STATUS: rollback test copy exactly matches the pre-change configuration implementation; working-tree MODIFIED_FILE remains changed; full tests, vet, relaykit build, and diff check pass.