Skip to content

feat: add passive channel monitoring mode #5592

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
seefs001 merged 4 commits into from
Jun 22, 2026
Merged

feat: add passive channel monitoring mode #5592

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
184d972
feat: add passive channel monitoring mode
exherb Jun 18, 2026
770cc8f
fix: clarify passive monitor mode copy
exherb Jun 18, 2026
091bb86
Merge remote-tracking branch 'upstream/main' into passive-channel-mon…
exherb Jun 22, 2026
7e5c6eb
Merge remote-tracking branch 'upstream/main' into passive-channel-mon…
exherb Jun 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 48 additions & 14 deletions controller/channel-test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -893,23 +893,14 @@ func TestChannel(c *gin.Context) {
var testAllChannelsLock sync.Mutex
var testAllChannelsRunning bool = false

func testAllChannels(notify bool) error {
testUserID, err := resolveChannelTestUserID(nil)
if err != nil {
return err
}

func testChannels(channels []*model.Channel, testUserID int, notify bool, allowDisable bool) error {
testAllChannelsLock.Lock()
if testAllChannelsRunning {
testAllChannelsLock.Unlock()
return errors.New("测试已在运行中")
}
testAllChannelsRunning = true
testAllChannelsLock.Unlock()
channels, getChannelErr := model.GetAllChannels(0, 0, true, false)
if getChannelErr != nil {
return getChannelErr
}
var disableThreshold = int64(common.ChannelDisableThreshold * 1000)
if disableThreshold == 0 {
disableThreshold = 10000000 // a impossible value
Expand Down Expand Up @@ -949,12 +940,12 @@ func testAllChannels(notify bool) error {
}

// disable channel
if isChannelEnabled && shouldBanChannel && channel.GetAutoBan() {
if allowDisable && isChannelEnabled && shouldBanChannel && channel.GetAutoBan() {
processChannelError(result.context, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError)
}

// enable channel
if !isChannelEnabled && service.ShouldEnableChannel(newAPIError, channel.Status) {
if result.localErr == nil && !isChannelEnabled && service.ShouldEnableChannel(newAPIError, channel.Status) {
service.EnableChannel(channel.Id, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.Name)
}

Expand All @@ -969,6 +960,44 @@ func testAllChannels(notify bool) error {
return nil
}

func selectChannelsForAutomaticTest(channels []*model.Channel, mode string) []*model.Channel {
selected := make([]*model.Channel, 0, len(channels))
for _, channel := range channels {
if channel.Status == common.ChannelStatusManuallyDisabled {
continue
}
if mode == operation_setting.ChannelTestModePassiveRecovery && channel.Status != common.ChannelStatusAutoDisabled {
continue
}
selected = append(selected, channel)
}
return selected
}

func testAllChannels(notify bool) error {
testUserID, err := resolveChannelTestUserID(nil)
if err != nil {
return err
}
channels, getChannelErr := model.GetAllChannels(0, 0, true, false)
if getChannelErr != nil {
return getChannelErr
}
return testChannels(selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModeScheduledAll), testUserID, notify, true)
}

func testAutoDisabledChannels(notify bool) error {
testUserID, err := resolveChannelTestUserID(nil)
if err != nil {
return err
}
channels, getChannelErr := model.GetAllChannels(0, 0, true, false)
if getChannelErr != nil {
return getChannelErr
}
return testChannels(selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModePassiveRecovery), testUserID, notify, false)
}

func TestAllChannels(c *gin.Context) {
err := testAllChannels(true)
if err != nil {
Expand Down Expand Up @@ -998,8 +1027,13 @@ func AutomaticallyTestChannels() {
frequency := operation_setting.GetMonitorSetting().AutoTestChannelMinutes
time.Sleep(time.Duration(int(math.Round(frequency))) * time.Minute)
common.SysLog(fmt.Sprintf("automatically test channels with interval %f minutes", frequency))
common.SysLog("automatically testing all channels")
_ = testAllChannels(false)
if operation_setting.GetMonitorSetting().ChannelTestMode == operation_setting.ChannelTestModePassiveRecovery {
common.SysLog("automatically testing auto-disabled channels")
_ = testAutoDisabledChannels(false)
} else {
common.SysLog("automatically testing all channels")
_ = testAllChannels(false)
}
common.SysLog("automatically channel test finished")
if !operation_setting.GetMonitorSetting().AutoTestChannelEnabled {
break
Expand Down
29 changes: 29 additions & 0 deletions controller/channel_test_internal_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import (

"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/dto"
"github.com/QuantumNous/new-api/model"
"github.com/QuantumNous/new-api/pkg/billingexpr"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/setting/operation_setting"
"github.com/QuantumNous/new-api/types"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -80,3 +82,30 @@ func TestResolveChannelTestUserIDUsesRequestUser(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 2, userID)
}

func TestSelectChannelsForAutomaticTestPassiveRecoveryOnlyUsesAutoDisabled(t *testing.T) {
channels := []*model.Channel{
{Id: 1, Status: common.ChannelStatusEnabled},
{Id: 2, Status: common.ChannelStatusAutoDisabled},
{Id: 3, Status: common.ChannelStatusManuallyDisabled},
}

selected := selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModePassiveRecovery)

require.Len(t, selected, 1)
require.Equal(t, 2, selected[0].Id)
}

func TestSelectChannelsForAutomaticTestScheduledSkipsManualDisabled(t *testing.T) {
channels := []*model.Channel{
{Id: 1, Status: common.ChannelStatusEnabled},
{Id: 2, Status: common.ChannelStatusAutoDisabled},
{Id: 3, Status: common.ChannelStatusManuallyDisabled},
}

selected := selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModeScheduledAll)

require.Len(t, selected, 2)
require.Equal(t, 1, selected[0].Id)
require.Equal(t, 2, selected[1].Id)
}
11 changes: 11 additions & 0 deletions setting/operation_setting/monitor_setting.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,19 @@ import (
type MonitorSetting struct {
AutoTestChannelEnabled bool `json:"auto_test_channel_enabled"`
AutoTestChannelMinutes float64 `json:"auto_test_channel_minutes"`
ChannelTestMode string `json:"channel_test_mode"`
}

const (
ChannelTestModeScheduledAll = "scheduled_all"
ChannelTestModePassiveRecovery = "passive_recovery"
)

// 默认配置
var monitorSetting = MonitorSetting{
AutoTestChannelEnabled: false,
AutoTestChannelMinutes: 10,
ChannelTestMode: ChannelTestModeScheduledAll,
}

func init() {
Expand All @@ -29,6 +36,7 @@ func GetMonitorSetting() *MonitorSetting {
if err == nil && frequency > 0 {
monitorSetting.AutoTestChannelEnabled = true
monitorSetting.AutoTestChannelMinutes = float64(frequency)
monitorSetting.ChannelTestMode = ChannelTestModeScheduledAll
}
}
if enabled, ok := os.LookupEnv("CHANNEL_TEST_ENABLED"); ok {
Expand All @@ -37,5 +45,8 @@ func GetMonitorSetting() *MonitorSetting {
monitorSetting.AutoTestChannelEnabled = parsed
}
}
if monitorSetting.ChannelTestMode != ChannelTestModePassiveRecovery {
monitorSetting.ChannelTestMode = ChannelTestModeScheduledAll
}
return &monitorSetting
}
1 change: 1 addition & 0 deletions web/default/src/features/models/components/drawers/model-mutate-drawer.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ export function ModelMutateDrawer({
'100-199,300-399,401-407,409-499,500-503,505-523,525-599',
'monitor_setting.auto_test_channel_enabled': false,
'monitor_setting.auto_test_channel_minutes': 10,
'monitor_setting.channel_test_mode': 'scheduled_all',
'channel_affinity_setting.enabled': false,
'channel_affinity_setting.switch_on_success': true,
'channel_affinity_setting.keep_on_channel_disabled': false,
Expand Down
1 change: 1 addition & 0 deletions web/default/src/features/system-settings/models/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ const defaultModelSettings: ModelSettings = {
'100-199,300-399,401-407,409-499,500-503,505-523,525-599',
'monitor_setting.auto_test_channel_enabled': false,
'monitor_setting.auto_test_channel_minutes': 10,
'monitor_setting.channel_test_mode': 'scheduled_all',
'channel_affinity_setting.enabled': false,
'channel_affinity_setting.switch_on_success': true,
'channel_affinity_setting.keep_on_channel_disabled': false,
Expand Down
78 changes: 77 additions & 1 deletion web/default/src/features/system-settings/models/routing-reliability-section.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ import {
FormMessage,
} from '@/components/ui/form'
import { Input } from '@/components/ui/input'
import {
Select,
SelectContent,
SelectGroup,
SelectItem,
SelectTrigger,
SelectValue,
} from '@/components/ui/select'
import { Separator } from '@/components/ui/separator'
import { Switch } from '@/components/ui/switch'
import { Textarea } from '@/components/ui/textarea'
Expand All @@ -53,6 +61,9 @@ const numericString = z.string().refine((value) => {
return !Number.isNaN(Number(trimmed)) && Number(trimmed) >= 0
}, 'Enter a non-negative number or leave empty')

const channelTestModes = ['scheduled_all', 'passive_recovery'] as const
type ChannelTestMode = (typeof channelTestModes)[number]

const routingReliabilitySchema = z
.object({
RetryTimes: z.coerce.number().min(0).max(10),
Expand All @@ -68,6 +79,7 @@ const routingReliabilitySchema = z
.number()
.int()
.min(1, 'Interval must be at least 1 minute'),
channel_test_mode: z.enum(channelTestModes),
}),
})
.superRefine((values, ctx) => {
Expand Down Expand Up @@ -112,6 +124,7 @@ type RoutingReliabilitySectionProps = {
AutomaticRetryStatusCodes: string
'monitor_setting.auto_test_channel_enabled': boolean
'monitor_setting.auto_test_channel_minutes': number
'monitor_setting.channel_test_mode': ChannelTestMode
}
}

Expand All @@ -129,6 +142,11 @@ type NormalizedRoutingReliabilityValues = {
AutomaticRetryStatusCodes: string
'monitor_setting.auto_test_channel_enabled': boolean
'monitor_setting.auto_test_channel_minutes': number
'monitor_setting.channel_test_mode': ChannelTestMode
}

function normalizeChannelTestMode(value?: string): ChannelTestMode {
return value === 'passive_recovery' ? 'passive_recovery' : 'scheduled_all'
}

const buildFormDefaults = (
Expand All @@ -148,6 +166,9 @@ const buildFormDefaults = (
defaults['monitor_setting.auto_test_channel_enabled'],
auto_test_channel_minutes:
defaults['monitor_setting.auto_test_channel_minutes'],
channel_test_mode: normalizeChannelTestMode(
defaults['monitor_setting.channel_test_mode']
),
},
})

Expand All @@ -171,6 +192,9 @@ const normalizeDefaults = (
defaults['monitor_setting.auto_test_channel_enabled'],
'monitor_setting.auto_test_channel_minutes':
defaults['monitor_setting.auto_test_channel_minutes'],
'monitor_setting.channel_test_mode': normalizeChannelTestMode(
defaults['monitor_setting.channel_test_mode']
),
})

const normalizeFormValues = (
Expand All @@ -193,6 +217,7 @@ const normalizeFormValues = (
values.monitor_setting.auto_test_channel_enabled,
'monitor_setting.auto_test_channel_minutes':
values.monitor_setting.auto_test_channel_minutes,
'monitor_setting.channel_test_mode': values.monitor_setting.channel_test_mode,
})

export function RoutingReliabilitySection({
Expand Down Expand Up @@ -222,6 +247,7 @@ export function RoutingReliabilitySection({

const autoDisableStatusCodes = form.watch('AutomaticDisableStatusCodes')
const autoRetryStatusCodes = form.watch('AutomaticRetryStatusCodes')
const channelTestMode = form.watch('monitor_setting.channel_test_mode')
const autoDisableParsed = useMemo(
() => parseHttpStatusCodeRules(autoDisableStatusCodes),
[autoDisableStatusCodes]
Expand Down Expand Up @@ -351,6 +377,52 @@ export function RoutingReliabilitySection({
)}
/>

<FormField
control={form.control}
name='monitor_setting.channel_test_mode'
render={({ field }) => (
<FormItem>
<FormLabel>{t('Channel test mode')}</FormLabel>
<Select
items={[
{
value: 'scheduled_all',
label: t('Scheduled full test'),
},
{
value: 'passive_recovery',
label: t('Passive recovery only'),
},
]}
value={field.value}
onValueChange={field.onChange}
>
<FormControl>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent alignItemWithTrigger={false}>
<SelectGroup>
<SelectItem value='scheduled_all'>
{t('Scheduled full test')}
</SelectItem>
<SelectItem value='passive_recovery'>
{t('Passive recovery only')}
</SelectItem>
</SelectGroup>
</SelectContent>
Comment thread
exherb marked this conversation as resolved.
</Select>
<FormDescription>
{t(
'Scheduled full test probes non-manually-disabled channels; passive recovery only checks auto-disabled channels after real request failures.'
)}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>

<FormField
control={form.control}
name='monitor_setting.auto_test_channel_minutes'
Expand All @@ -366,7 +438,11 @@ export function RoutingReliabilitySection({
/>
</FormControl>
<FormDescription>
{t('How frequently the system tests all channels')}
{channelTestMode === 'passive_recovery'
? t(
'How frequently the system checks auto-disabled channels for recovery'
)
: t('How frequently the system tests all channels')}
</FormDescription>
<FormMessage />
</FormItem>
Expand Down
2 changes: 2 additions & 0 deletions web/default/src/features/system-settings/models/section-registry.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ const MODELS_SECTIONS = [
settings['monitor_setting.auto_test_channel_enabled'],
'monitor_setting.auto_test_channel_minutes':
settings['monitor_setting.auto_test_channel_minutes'],
'monitor_setting.channel_test_mode':
settings['monitor_setting.channel_test_mode'],
}}
/>
),
Expand Down
1 change: 1 addition & 0 deletions web/default/src/features/system-settings/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,7 @@ export type ModelSettings = {
AutomaticRetryStatusCodes: string
'monitor_setting.auto_test_channel_enabled': boolean
'monitor_setting.auto_test_channel_minutes': number
'monitor_setting.channel_test_mode': 'scheduled_all' | 'passive_recovery'
'channel_affinity_setting.enabled': boolean
'channel_affinity_setting.switch_on_success': boolean
'channel_affinity_setting.keep_on_channel_disabled': boolean
Expand Down
Loading