feat: add passive channel monitoring mode

controller/channel-test.go

@@ -893,23 +893,14 @@ func TestChannel(c *gin.Context) {
 var testAllChannelsLock sync.Mutex
 var testAllChannelsRunning bool = false
 
-func testAllChannels(notify bool) error {
-	testUserID, err := resolveChannelTestUserID(nil)
-	if err != nil {
-		return err
-	}
-
+func testChannels(channels []*model.Channel, testUserID int, notify bool, allowDisable bool) error {
 	testAllChannelsLock.Lock()
 	if testAllChannelsRunning {
 		testAllChannelsLock.Unlock()
 		return errors.New("测试已在运行中")
 	}
 	testAllChannelsRunning = true
 	testAllChannelsLock.Unlock()
-	channels, getChannelErr := model.GetAllChannels(0, 0, true, false)
-	if getChannelErr != nil {
-		return getChannelErr
-	}
 	var disableThreshold = int64(common.ChannelDisableThreshold * 1000)
 	if disableThreshold == 0 {
 		disableThreshold = 10000000 // a impossible value
@@ -949,12 +940,12 @@ func testAllChannels(notify bool) error {
 			}
 
 			// disable channel
-			if isChannelEnabled && shouldBanChannel && channel.GetAutoBan() {
+			if allowDisable && isChannelEnabled && shouldBanChannel && channel.GetAutoBan() {
 				processChannelError(result.context, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError)
 			}
 
 			// enable channel
-			if !isChannelEnabled && service.ShouldEnableChannel(newAPIError, channel.Status) {
+			if result.localErr == nil && !isChannelEnabled && service.ShouldEnableChannel(newAPIError, channel.Status) {
 				service.EnableChannel(channel.Id, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.Name)
 			}
 
@@ -969,6 +960,44 @@ func testAllChannels(notify bool) error {
 	return nil
 }
 
+func selectChannelsForAutomaticTest(channels []*model.Channel, mode string) []*model.Channel {
+	selected := make([]*model.Channel, 0, len(channels))
+	for _, channel := range channels {
+		if channel.Status == common.ChannelStatusManuallyDisabled {
+			continue
+		}
+		if mode == operation_setting.ChannelTestModePassiveRecovery && channel.Status != common.ChannelStatusAutoDisabled {
+			continue
+		}
+		selected = append(selected, channel)
+	}
+	return selected
+}
+
+func testAllChannels(notify bool) error {
+	testUserID, err := resolveChannelTestUserID(nil)
+	if err != nil {
+		return err
+	}
+	channels, getChannelErr := model.GetAllChannels(0, 0, true, false)
+	if getChannelErr != nil {
+		return getChannelErr
+	}
+	return testChannels(selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModeScheduledAll), testUserID, notify, true)
+}
+
+func testAutoDisabledChannels(notify bool) error {
+	testUserID, err := resolveChannelTestUserID(nil)
+	if err != nil {
+		return err
+	}
+	channels, getChannelErr := model.GetAllChannels(0, 0, true, false)
+	if getChannelErr != nil {
+		return getChannelErr
+	}
+	return testChannels(selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModePassiveRecovery), testUserID, notify, false)
+}
+
 func TestAllChannels(c *gin.Context) {
 	err := testAllChannels(true)
 	if err != nil {
@@ -998,8 +1027,13 @@ func AutomaticallyTestChannels() {
 				frequency := operation_setting.GetMonitorSetting().AutoTestChannelMinutes
 				time.Sleep(time.Duration(int(math.Round(frequency))) * time.Minute)
 				common.SysLog(fmt.Sprintf("automatically test channels with interval %f minutes", frequency))
-				common.SysLog("automatically testing all channels")
-				_ = testAllChannels(false)
+				if operation_setting.GetMonitorSetting().ChannelTestMode == operation_setting.ChannelTestModePassiveRecovery {
+					common.SysLog("automatically testing auto-disabled channels")
+					_ = testAutoDisabledChannels(false)
+				} else {
+					common.SysLog("automatically testing all channels")
+					_ = testAllChannels(false)
+				}
 				common.SysLog("automatically channel test finished")
 				if !operation_setting.GetMonitorSetting().AutoTestChannelEnabled {
 					break

controller/channel_test_internal_test.go

@@ -6,8 +6,10 @@ import (
 
 	"github.com/QuantumNous/new-api/common"
 	"github.com/QuantumNous/new-api/dto"
+	"github.com/QuantumNous/new-api/model"
 	"github.com/QuantumNous/new-api/pkg/billingexpr"
 	relaycommon "github.com/QuantumNous/new-api/relay/common"
+	"github.com/QuantumNous/new-api/setting/operation_setting"
 	"github.com/QuantumNous/new-api/types"
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/require"
@@ -80,3 +82,30 @@ func TestResolveChannelTestUserIDUsesRequestUser(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, 2, userID)
 }
+
+func TestSelectChannelsForAutomaticTestPassiveRecoveryOnlyUsesAutoDisabled(t *testing.T) {
+	channels := []*model.Channel{
+		{Id: 1, Status: common.ChannelStatusEnabled},
+		{Id: 2, Status: common.ChannelStatusAutoDisabled},
+		{Id: 3, Status: common.ChannelStatusManuallyDisabled},
+	}
+
+	selected := selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModePassiveRecovery)
+
+	require.Len(t, selected, 1)
+	require.Equal(t, 2, selected[0].Id)
+}
+
+func TestSelectChannelsForAutomaticTestScheduledSkipsManualDisabled(t *testing.T) {
+	channels := []*model.Channel{
+		{Id: 1, Status: common.ChannelStatusEnabled},
+		{Id: 2, Status: common.ChannelStatusAutoDisabled},
+		{Id: 3, Status: common.ChannelStatusManuallyDisabled},
+	}
+
+	selected := selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModeScheduledAll)
+
+	require.Len(t, selected, 2)
+	require.Equal(t, 1, selected[0].Id)
+	require.Equal(t, 2, selected[1].Id)
+}

setting/operation_setting/monitor_setting.go

@@ -10,12 +10,19 @@ import (
 type MonitorSetting struct {
 	AutoTestChannelEnabled bool    `json:"auto_test_channel_enabled"`
 	AutoTestChannelMinutes float64 `json:"auto_test_channel_minutes"`
+	ChannelTestMode        string  `json:"channel_test_mode"`
 }
 
+const (
+	ChannelTestModeScheduledAll    = "scheduled_all"
+	ChannelTestModePassiveRecovery = "passive_recovery"
+)
+
 // 默认配置
 var monitorSetting = MonitorSetting{
 	AutoTestChannelEnabled: false,
 	AutoTestChannelMinutes: 10,
+	ChannelTestMode:        ChannelTestModeScheduledAll,
 }
 
 func init() {
@@ -29,7 +36,11 @@ func GetMonitorSetting() *MonitorSetting {
 		if err == nil && frequency > 0 {
 			monitorSetting.AutoTestChannelEnabled = true
 			monitorSetting.AutoTestChannelMinutes = float64(frequency)
+			monitorSetting.ChannelTestMode = ChannelTestModeScheduledAll
 		}
 	}
+	if monitorSetting.ChannelTestMode != ChannelTestModePassiveRecovery {
+		monitorSetting.ChannelTestMode = ChannelTestModeScheduledAll
+	}
 	return &monitorSetting
 }

web/default/src/features/models/components/drawers/model-mutate-drawer.tsx

@@ -210,6 +210,7 @@ export function ModelMutateDrawer({
         '100-199,300-399,401-407,409-499,500-503,505-523,525-599',
       'monitor_setting.auto_test_channel_enabled': false,
       'monitor_setting.auto_test_channel_minutes': 10,
+      'monitor_setting.channel_test_mode': 'scheduled_all',
       'channel_affinity_setting.enabled': false,
       'channel_affinity_setting.switch_on_success': true,
       'channel_affinity_setting.keep_on_channel_disabled': false,

web/default/src/features/system-settings/models/index.tsx

@@ -72,6 +72,7 @@ const defaultModelSettings: ModelSettings = {
     '100-199,300-399,401-407,409-499,500-503,505-523,525-599',
   'monitor_setting.auto_test_channel_enabled': false,
   'monitor_setting.auto_test_channel_minutes': 10,
+  'monitor_setting.channel_test_mode': 'scheduled_all',
   'channel_affinity_setting.enabled': false,
   'channel_affinity_setting.switch_on_success': true,
   'channel_affinity_setting.keep_on_channel_disabled': false,

web/default/src/features/system-settings/models/routing-reliability-section.tsx

@@ -33,6 +33,14 @@ import {
   FormMessage,
 } from '@/components/ui/form'
 import { Input } from '@/components/ui/input'
+import {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from '@/components/ui/select'
 import { Separator } from '@/components/ui/separator'
 import { Switch } from '@/components/ui/switch'
 import { Textarea } from '@/components/ui/textarea'
@@ -53,6 +61,9 @@ const numericString = z.string().refine((value) => {
   return !Number.isNaN(Number(trimmed)) && Number(trimmed) >= 0
 }, 'Enter a non-negative number or leave empty')
 
+const channelTestModes = ['scheduled_all', 'passive_recovery'] as const
+type ChannelTestMode = (typeof channelTestModes)[number]
+
 const routingReliabilitySchema = z
   .object({
     RetryTimes: z.coerce.number().min(0).max(10),
@@ -68,6 +79,7 @@ const routingReliabilitySchema = z
         .number()
         .int()
         .min(1, 'Interval must be at least 1 minute'),
+      channel_test_mode: z.enum(channelTestModes),
     }),
   })
   .superRefine((values, ctx) => {
@@ -112,6 +124,7 @@ type RoutingReliabilitySectionProps = {
     AutomaticRetryStatusCodes: string
     'monitor_setting.auto_test_channel_enabled': boolean
     'monitor_setting.auto_test_channel_minutes': number
+    'monitor_setting.channel_test_mode': ChannelTestMode
   }
 }
 
@@ -129,6 +142,11 @@ type NormalizedRoutingReliabilityValues = {
   AutomaticRetryStatusCodes: string
   'monitor_setting.auto_test_channel_enabled': boolean
   'monitor_setting.auto_test_channel_minutes': number
+  'monitor_setting.channel_test_mode': ChannelTestMode
+}
+
+function normalizeChannelTestMode(value?: string): ChannelTestMode {
+  return value === 'passive_recovery' ? 'passive_recovery' : 'scheduled_all'
 }
 
 const buildFormDefaults = (
@@ -148,6 +166,9 @@ const buildFormDefaults = (
       defaults['monitor_setting.auto_test_channel_enabled'],
     auto_test_channel_minutes:
       defaults['monitor_setting.auto_test_channel_minutes'],
+    channel_test_mode: normalizeChannelTestMode(
+      defaults['monitor_setting.channel_test_mode']
+    ),
   },
 })
 
@@ -171,6 +192,9 @@ const normalizeDefaults = (
     defaults['monitor_setting.auto_test_channel_enabled'],
   'monitor_setting.auto_test_channel_minutes':
     defaults['monitor_setting.auto_test_channel_minutes'],
+  'monitor_setting.channel_test_mode': normalizeChannelTestMode(
+    defaults['monitor_setting.channel_test_mode']
+  ),
 })
 
 const normalizeFormValues = (
@@ -193,6 +217,7 @@ const normalizeFormValues = (
     values.monitor_setting.auto_test_channel_enabled,
   'monitor_setting.auto_test_channel_minutes':
     values.monitor_setting.auto_test_channel_minutes,
+  'monitor_setting.channel_test_mode': values.monitor_setting.channel_test_mode,
 })
 
 export function RoutingReliabilitySection({
@@ -222,6 +247,7 @@ export function RoutingReliabilitySection({
 
   const autoDisableStatusCodes = form.watch('AutomaticDisableStatusCodes')
   const autoRetryStatusCodes = form.watch('AutomaticRetryStatusCodes')
+  const channelTestMode = form.watch('monitor_setting.channel_test_mode')
   const autoDisableParsed = useMemo(
     () => parseHttpStatusCodeRules(autoDisableStatusCodes),
     [autoDisableStatusCodes]
@@ -351,6 +377,52 @@ export function RoutingReliabilitySection({
                 )}
               />
 
+              <FormField
+                control={form.control}
+                name='monitor_setting.channel_test_mode'
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>{t('Channel test mode')}</FormLabel>
+                    <Select
+                      items={[
+                        {
+                          value: 'scheduled_all',
+                          label: t('Scheduled full test'),
+                        },
+                        {
+                          value: 'passive_recovery',
+                          label: t('Passive recovery only'),
+                        },
+                      ]}
+                      value={field.value}
+                      onValueChange={field.onChange}
+                    >
+                      <FormControl>
+                        <SelectTrigger>
+                          <SelectValue />
+                        </SelectTrigger>
+                      </FormControl>
+                      <SelectContent alignItemWithTrigger={false}>
+                        <SelectGroup>
+                          <SelectItem value='scheduled_all'>
+                            {t('Scheduled full test')}
+                          </SelectItem>
+                          <SelectItem value='passive_recovery'>
+                            {t('Passive recovery only')}
+                          </SelectItem>
+                        </SelectGroup>
+                      </SelectContent>
+                    </Select>
+                    <FormDescription>
+                      {t(
+                        'Scheduled full test probes non-manually-disabled channels; passive recovery only checks auto-disabled channels after real request failures.'
+                      )}
+                    </FormDescription>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+
               <FormField
                 control={form.control}
                 name='monitor_setting.auto_test_channel_minutes'
@@ -366,7 +438,11 @@ export function RoutingReliabilitySection({
                       />
                     </FormControl>
                     <FormDescription>
-                      {t('How frequently the system tests all channels')}
+                      {channelTestMode === 'passive_recovery'
+                        ? t(
+                            'How frequently the system checks auto-disabled channels for recovery'
+                          )
+                        : t('How frequently the system tests all channels')}
                     </FormDescription>
                     <FormMessage />
                   </FormItem>

web/default/src/features/system-settings/models/section-registry.tsx

@@ -83,6 +83,8 @@ const MODELS_SECTIONS = [
             settings['monitor_setting.auto_test_channel_enabled'],
           'monitor_setting.auto_test_channel_minutes':
             settings['monitor_setting.auto_test_channel_minutes'],
+          'monitor_setting.channel_test_mode':
+            settings['monitor_setting.channel_test_mode'],
         }}
       />
     ),

web/default/src/features/system-settings/types.ts

@@ -183,6 +183,7 @@ export type ModelSettings = {
   AutomaticRetryStatusCodes: string
   'monitor_setting.auto_test_channel_enabled': boolean
   'monitor_setting.auto_test_channel_minutes': number
+  'monitor_setting.channel_test_mode': 'scheduled_all' | 'passive_recovery'
   'channel_affinity_setting.enabled': boolean
   'channel_affinity_setting.switch_on_success': boolean
   'channel_affinity_setting.keep_on_channel_disabled': boolean

web/default/src/i18n/locales/en.json

@@ -704,6 +704,7 @@
     "Channel models": "Channel models",
     "Channel name is required": "Channel name is required",
     "Channel test completed": "Channel test completed",
+    "Channel test mode": "Channel test mode",
     "Channel type is required": "Channel type is required",
     "Channel updated successfully": "Channel updated successfully",
     "Channel-specific settings (JSON format)": "Channel-specific settings (JSON format)",
@@ -2078,6 +2079,7 @@
     "Hourly token usage by model over the past 24 hours": "Hourly token usage by model over the past 24 hours",
     "hours": "hours",
     "How client credentials are sent to the token endpoint": "How client credentials are sent to the token endpoint",
+    "How frequently the system checks auto-disabled channels for recovery": "How frequently the system checks auto-disabled channels for recovery",
     "How frequently the system tests all channels": "How frequently the system tests all channels",
     "How It Works": "How It Works",
     "How model mapping works": "How model mapping works",
@@ -3014,6 +3016,7 @@
     "Pass when key is missing": "Pass when key is missing",
     "Pass-Through": "Pass-Through",
     "Pass-through Headers (comma-separated or JSON array)": "Pass-through Headers (comma-separated or JSON array)",
+    "Passive recovery only": "Passive recovery only",
     "Passkey": "Passkey",
     "Passkey Authentication": "Passkey Authentication",
     "Passkey is not available in this browser": "Passkey is not available in this browser",
@@ -3690,6 +3693,8 @@
     "Scan this QR code with your authenticator app (Google Authenticator, Microsoft Authenticator, etc.)": "Scan this QR code with your authenticator app (Google Authenticator, Microsoft Authenticator, etc.)",
     "Scenario Templates": "Scenario Templates",
     "Scheduled channel tests": "Scheduled channel tests",
+    "Scheduled full test": "Scheduled full test",
+    "Scheduled full test probes non-manually-disabled channels; passive recovery only checks auto-disabled channels after real request failures.": "Scheduled full test probes non-manually-disabled channels; passive recovery only checks auto-disabled channels after real request failures.",
     "scheduling controls": "scheduling controls",
     "Science": "Science",
     "Scope": "Scope",