fix(channel): recover auto-disabled multi-key channels

controller/channel-test.go

@@ -69,7 +69,7 @@ func resolveChannelTestUserID(c *gin.Context) (int, error) {
 	return rootUser.Id, nil
 }
 
-func testChannel(ctx context.Context, channel *model.Channel, testUserID int, testModel string, endpointType string, isStream bool) testResult {
+func testChannel(ctx context.Context, channel *model.Channel, testUserID int, testModel string, endpointType string, isStream bool, keyIndex *int) testResult {
 	if ctx == nil {
 		ctx = context.Background()
 	}
@@ -168,7 +168,12 @@ func testChannel(ctx context.Context, channel *model.Channel, testUserID int, te
 	group, _ := model.GetUserGroup(testUserID, false)
 	c.Set("group", group)
 
-	newAPIError := middleware.SetupContextForSelectedChannel(c, channel, testModel)
+	var newAPIError *types.NewAPIError
+	if keyIndex == nil {
+		newAPIError = middleware.SetupContextForSelectedChannel(c, channel, testModel)
+	} else {
+		newAPIError = middleware.SetupContextForSelectedChannelKey(c, channel, testModel, *keyIndex)
+	}
 	if newAPIError != nil {
 		return testResult{
 			context:     c,
@@ -867,7 +872,7 @@ func TestChannel(c *gin.Context) {
 	if c.Request != nil {
 		requestCtx = c.Request.Context()
 	}
-	result := testChannel(requestCtx, channel, testUserID, testModel, endpointType, isStream)
+	result := testChannel(requestCtx, channel, testUserID, testModel, endpointType, isStream, nil)
 	if result.localErr != nil {
 		resp := gin.H{
 			"success": false,
@@ -911,10 +916,64 @@ type channelTestSummary struct {
 }
 
 func testChannelForHealthCheck(ctx context.Context, channel *model.Channel, testUserID int, allowDisable bool, disableThreshold int64) channelTestSummary {
+	// Normal selection cannot choose a key after every key has been auto-disabled,
+	// so recovery checks probe those keys explicitly and leave manual disables alone.
+	if channel.Status == common.ChannelStatusAutoDisabled && channel.ChannelInfo.IsMultiKey {
+		keyIndexes := multiKeyRecoveryIndexes(channel)
+		if len(keyIndexes) > 0 {
+			keySummary := channelTestSummary{}
+			for _, keyIndex := range keyIndexes {
+				result := testChannelKeyForHealthCheck(ctx, channel, testUserID, allowDisable, disableThreshold, &keyIndex)
+				keySummary.Tested += result.Tested
+				keySummary.Succeeded += result.Succeeded
+				keySummary.Failed += result.Failed
+				keySummary.Disabled += result.Disabled
+				keySummary.Enabled += result.Enabled
+				if ctx.Err() != nil {
+					break
+				}
+			}
+			summary := channelTestSummary{}
+			if keySummary.Tested > 0 {
+				summary.Tested = 1
+				if keySummary.Succeeded > 0 {
+					summary.Succeeded = 1
+				} else {
+					summary.Failed = 1
+				}
+			}
+			if keySummary.Disabled > 0 {
+				summary.Disabled = 1
+			}
+			if keySummary.Enabled > 0 {
+				summary.Enabled = 1
+			}
+			return summary
+		}
+	}
+	return testChannelKeyForHealthCheck(ctx, channel, testUserID, allowDisable, disableThreshold, nil)
+}
+
+func multiKeyRecoveryIndexes(channel *model.Channel) []int {
+	if channel == nil || channel.Status != common.ChannelStatusAutoDisabled ||
+		!channel.ChannelInfo.IsMultiKey || !channel.ChannelInfo.MultiKeyAutoRecovery {
+		return nil
+	}
+	keys := channel.GetKeys()
+	indexes := make([]int, 0, len(keys))
+	for index := range keys {
+		if channel.ChannelInfo.MultiKeyStatusList[index] == common.ChannelStatusAutoDisabled {
+			indexes = append(indexes, index)
+		}
+	}
+	return indexes
+}
+
+func testChannelKeyForHealthCheck(ctx context.Context, channel *model.Channel, testUserID int, allowDisable bool, disableThreshold int64, keyIndex *int) channelTestSummary {
 	summary := channelTestSummary{}
 	isChannelEnabled := channel.Status == common.ChannelStatusEnabled
 	tik := time.Now()
-	result := testChannel(ctx, channel, testUserID, "", "", shouldUseStreamForAutomaticChannelTest(channel))
+	result := testChannel(ctx, channel, testUserID, "", "", shouldUseStreamForAutomaticChannelTest(channel), keyIndex)
 	milliseconds := time.Since(tik).Milliseconds()
 	if ctx.Err() != nil {
 		return summary

controller/channel.go

@@ -589,6 +589,7 @@ func RefreshCodexChannelCredential(c *gin.Context) {
 type AddChannelRequest struct {
 	Mode                      string                `json:"mode"`
 	MultiKeyMode              constant.MultiKeyMode `json:"multi_key_mode"`
+	MultiKeyAutoRecovery      bool                  `json:"multi_key_auto_recovery"`
 	BatchAddSetKeyPrefix2Name bool                  `json:"batch_add_set_key_prefix_2_name"`
 	Channel                   *model.Channel        `json:"channel"`
 }
@@ -657,6 +658,7 @@ func AddChannel(c *gin.Context) {
 	case "multi_to_single":
 		addChannelRequest.Channel.ChannelInfo.IsMultiKey = true
 		addChannelRequest.Channel.ChannelInfo.MultiKeyMode = addChannelRequest.MultiKeyMode
+		addChannelRequest.Channel.ChannelInfo.MultiKeyAutoRecovery = addChannelRequest.MultiKeyAutoRecovery
 		if addChannelRequest.Channel.Type == constant.ChannelTypeVertexAi && addChannelRequest.Channel.GetOtherSettings().VertexKeyType != dto.VertexKeyTypeAPIKey {
 			array, err := getVertexArrayKeys(addChannelRequest.Channel.Key)
 			if err != nil {
@@ -954,8 +956,9 @@ func DeleteChannelBatch(c *gin.Context) {
 
 type PatchChannel struct {
 	model.Channel
-	MultiKeyMode *string `json:"multi_key_mode"`
-	KeyMode      *string `json:"key_mode"` // 多key模式下密钥覆盖或者追加
+	MultiKeyMode         *string `json:"multi_key_mode"`
+	MultiKeyAutoRecovery *bool   `json:"multi_key_auto_recovery"`
+	KeyMode              *string `json:"key_mode"` // 多key模式下密钥覆盖或者追加
 }
 
 type ChannelStatusRequest struct {
@@ -1036,6 +1039,9 @@ func UpdateChannel(c *gin.Context) {
 	if channel.MultiKeyMode != nil && *channel.MultiKeyMode != "" {
 		channel.ChannelInfo.MultiKeyMode = constant.MultiKeyMode(*channel.MultiKeyMode)
 	}
+	if channel.MultiKeyAutoRecovery != nil && channel.ChannelInfo.IsMultiKey {
+		channel.ChannelInfo.MultiKeyAutoRecovery = *channel.MultiKeyAutoRecovery
+	}
 
 	// 处理多key模式下的密钥追加/覆盖逻辑
 	if channel.KeyMode != nil && channel.ChannelInfo.IsMultiKey {

controller/channel_authz.go

@@ -118,19 +118,20 @@ func clearChannelReadOnlyFields(channel *PatchChannel, requestData map[string]an
 // to the fail-closed branch and is treated as sensitive. The
 // TestChannelFieldsAreClassified guard test enforces this.
 var channelNonSensitiveFields = map[string]struct{}{
-	"id":                  {},
-	"test_model":          {},
-	"name":                {},
-	"weight":              {},
-	"models":              {},
-	"group":               {},
-	"model_mapping":       {},
-	"status_code_mapping": {},
-	"priority":            {},
-	"auto_ban":            {},
-	"other_info":          {},
-	"tag":                 {},
-	"remark":              {},
-	"channel_info":        {},
-	"multi_key_mode":      {},
+	"id":                      {},
+	"test_model":              {},
+	"name":                    {},
+	"weight":                  {},
+	"models":                  {},
+	"group":                   {},
+	"model_mapping":           {},
+	"status_code_mapping":     {},
+	"priority":                {},
+	"auto_ban":                {},
+	"other_info":              {},
+	"tag":                     {},
+	"remark":                  {},
+	"channel_info":            {},
+	"multi_key_mode":          {},
+	"multi_key_auto_recovery": {},
 }

controller/channel_test_internal_test.go

@@ -341,6 +341,43 @@ func TestSelectChannelsForAutomaticTestAutoBanOnlyUsesEligibleChannels(t *testin
 	require.Equal(t, 3, selected[1].Id)
 }
 
+func TestMultiKeyRecoveryIndexesSkipsManuallyDisabledKeys(t *testing.T) {
+	channel := &model.Channel{
+		Status: common.ChannelStatusAutoDisabled,
+		Key:    "key-a\nkey-b\nkey-c",
+		ChannelInfo: model.ChannelInfo{
+			IsMultiKey:           true,
+			MultiKeyAutoRecovery: true,
+			MultiKeyStatusList: map[int]int{
+				0: common.ChannelStatusAutoDisabled,
+				1: common.ChannelStatusManuallyDisabled,
+			},
+		},
+	}
+
+	indexes := multiKeyRecoveryIndexes(channel)
+
+	assert.Equal(t, []int{0}, indexes)
+}
+
+func TestAutoDisabledMultiKeyRecoveryRequiresChannelOption(t *testing.T) {
+	channel := &model.Channel{
+		Status: common.ChannelStatusAutoDisabled,
+		Key:    "key-a\nkey-b",
+		ChannelInfo: model.ChannelInfo{
+			IsMultiKey: true,
+			MultiKeyStatusList: map[int]int{
+				0: common.ChannelStatusAutoDisabled,
+				1: common.ChannelStatusAutoDisabled,
+			},
+		},
+	}
+
+	assert.Empty(t, multiKeyRecoveryIndexes(channel))
+	channel.ChannelInfo.MultiKeyAutoRecovery = true
+	assert.Equal(t, []int{0, 1}, multiKeyRecoveryIndexes(channel))
+}
+
 func TestRunChannelTestWorkersHonorsConfiguredConcurrency(t *testing.T) {
 	originalInterval := common.RequestInterval
 	common.RequestInterval = 0

middleware/distributor.go

@@ -577,6 +577,16 @@ func getTaskOriginModelName(c *gin.Context) string {
 }
 
 func SetupContextForSelectedChannel(c *gin.Context, channel *model.Channel, modelName string) *types.NewAPIError {
+	return setupContextForSelectedChannel(c, channel, modelName, nil)
+}
+
+// SetupContextForSelectedChannelKey is used by channel health checks that must
+// probe a specific disabled key without changing normal request key selection.
+func SetupContextForSelectedChannelKey(c *gin.Context, channel *model.Channel, modelName string, keyIndex int) *types.NewAPIError {
+	return setupContextForSelectedChannel(c, channel, modelName, &keyIndex)
+}
+
+func setupContextForSelectedChannel(c *gin.Context, channel *model.Channel, modelName string, keyIndex *int) *types.NewAPIError {
 	c.Set("original_model", modelName) // for retry
 	expectedPlugin := c.GetString("expected_task_plugin_key")
 	if channel == nil {
@@ -640,9 +650,21 @@ func SetupContextForSelectedChannel(c *gin.Context, channel *model.Channel, mode
 	common.SetContextKey(c, constant.ContextKeyChannelModelMapping, channel.GetModelMapping())
 	common.SetContextKey(c, constant.ContextKeyChannelStatusCodeMapping, channel.GetStatusCodeMapping())
 
-	key, index, newAPIError := channel.GetNextEnabledKey()
-	if newAPIError != nil {
-		return newAPIError
+	key := ""
+	index := 0
+	if keyIndex == nil {
+		var newAPIError *types.NewAPIError
+		key, index, newAPIError = channel.GetNextEnabledKey()
+		if newAPIError != nil {
+			return newAPIError
+		}
+	} else {
+		keys := channel.GetKeys()
+		if !channel.ChannelInfo.IsMultiKey || *keyIndex < 0 || *keyIndex >= len(keys) {
+			return types.NewError(errors.New("invalid channel key index"), types.ErrorCodeChannelNoAvailableKey)
+		}
+		key = keys[*keyIndex]
+		index = *keyIndex
 	}
 	if channel.ChannelInfo.IsMultiKey {
 		common.SetContextKey(c, constant.ContextKeyChannelIsMultiKey, true)

middleware/distributor_test.go

@@ -2,8 +2,10 @@ package middleware
 
 import (
 	"fmt"
+	"net/http/httptest"
 	"testing"
 
+	"github.com/QuantumNous/new-api/common"
 	"github.com/QuantumNous/new-api/constant"
 	"github.com/QuantumNous/new-api/model"
 	"github.com/QuantumNous/new-api/pkg/jsplugin"
@@ -147,3 +149,44 @@ export const protocols = {openai_responses: {
 }};
 `, key, key, channelType)
 }
+
+func TestSetupContextForSelectedChannelKeyCanSelectAutoDisabledKey(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	channel := &model.Channel{
+		Key: "key-a\nkey-b",
+		ChannelInfo: model.ChannelInfo{
+			IsMultiKey: true,
+			MultiKeyStatusList: map[int]int{
+				0: common.ChannelStatusAutoDisabled,
+				1: common.ChannelStatusAutoDisabled,
+			},
+		},
+	}
+
+	ctx, _ := gin.CreateTestContext(httptest.NewRecorder())
+	err := SetupContextForSelectedChannelKey(ctx, channel, "gpt-test", 1)
+
+	require.Nil(t, err)
+	assert.Equal(t, "key-b", common.GetContextKeyString(ctx, constant.ContextKeyChannelKey))
+	assert.Equal(t, 1, common.GetContextKeyInt(ctx, constant.ContextKeyChannelMultiKeyIndex))
+}
+
+func TestSetupContextForSelectedChannelStillRejectsAllDisabledKeys(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	channel := &model.Channel{
+		Key: "key-a\nkey-b",
+		ChannelInfo: model.ChannelInfo{
+			IsMultiKey: true,
+			MultiKeyStatusList: map[int]int{
+				0: common.ChannelStatusAutoDisabled,
+				1: common.ChannelStatusAutoDisabled,
+			},
+		},
+	}
+
+	ctx, _ := gin.CreateTestContext(httptest.NewRecorder())
+	err := SetupContextForSelectedChannel(ctx, channel, "gpt-test")
+
+	require.NotNil(t, err)
+	assert.Contains(t, err.Error(), "no enabled keys")
+}

model/channel.go

@@ -67,6 +67,7 @@ type ChannelInfo struct {
 	MultiKeyDisabledTime   map[int]int64         `json:"multi_key_disabled_time,omitempty"`   // key禁用时间列表，key index -> time
 	MultiKeyPollingIndex   int                   `json:"multi_key_polling_index"`             // 多Key模式下轮询的key索引
 	MultiKeyMode           constant.MultiKeyMode `json:"multi_key_mode"`
+	MultiKeyAutoRecovery   bool                  `json:"multi_key_auto_recovery,omitempty"`
 }
 
 type ChannelSortOptions struct {
@@ -782,7 +783,7 @@ func UpdateChannelStatus(channelId int, usingKey string, status int, reason stri
 	if err != nil {
 		return false
 	} else {
-		if channel.Status == status {
+		if channel.Status == status && (!channel.ChannelInfo.IsMultiKey || usingKey == "") {
 			return false
 		}
 

model/channel_status_test.go

@@ -51,6 +51,33 @@ func TestUpdateChannelStatusPersistsMultiKeyState(t *testing.T) {
 	assert.Equal(t, 1, stored.ChannelInfo.MultiKeyPollingIndex)
 }
 
+func TestUpdateChannelStatusEnablesRemainingKeyAfterChannelRecovery(t *testing.T) {
+	setupChannelStatusTest(t)
+
+	channel := Channel{
+		Name:   "multi-key-recovery",
+		Key:    "key-a\nkey-b",
+		Status: common.ChannelStatusAutoDisabled,
+		ChannelInfo: ChannelInfo{
+			IsMultiKey:   true,
+			MultiKeySize: 2,
+			MultiKeyStatusList: map[int]int{
+				0: common.ChannelStatusAutoDisabled,
+				1: common.ChannelStatusAutoDisabled,
+			},
+		},
+	}
+	require.NoError(t, DB.Create(&channel).Error)
+
+	require.True(t, UpdateChannelStatus(channel.Id, "key-a", common.ChannelStatusEnabled, ""))
+	require.True(t, UpdateChannelStatus(channel.Id, "key-b", common.ChannelStatusEnabled, ""))
+
+	var stored Channel
+	require.NoError(t, DB.First(&stored, channel.Id).Error)
+	assert.Equal(t, common.ChannelStatusEnabled, stored.Status)
+	assert.Empty(t, stored.ChannelInfo.MultiKeyStatusList)
+}
+
 func TestSaveStatusStateFromSingleKeySnapshotPreservesUnownedColumns(t *testing.T) {
 	setupChannelStatusTest(t)
 

web/src/features/channels/api.ts

@@ -38,6 +38,7 @@ import type {
   SearchChannelsParams,
   SearchChannelsResponse,
   TagOperationParams,
+  UpdateChannelRequest,
 } from './types'
 
 const channelActionConfig = (
@@ -139,7 +140,7 @@ export async function createChannel(
  */
 export async function updateChannel(
   id: number,
-  data: Partial<Channel>
+  data: UpdateChannelRequest
 ): Promise<{ success: boolean; message?: string; data?: Channel }> {
   const res = await api.put(
     '/api/channel/',