Skip to content

feat: add per-channel health check settings #6283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
DawnMoon1542 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat: add per-channel health check settings #6283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5a6892f
feat: add per-channel health check settings
DawnMoon1542 Jul 18, 2026
164f470
fix: keep health check form state aligned during refresh and save
DawnMoon1542 Jul 18, 2026
b487001
fix: 补齐渠道健康检查表格无障碍标签与本地化文案
DawnMoon1542 Jul 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 23 additions & 15 deletions controller/channel-test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -659,10 +659,6 @@ func validateTestResponseBody(respBody []byte, isStream bool) error {
return nil
}

func shouldUseStreamForAutomaticChannelTest(channel *model.Channel) bool {
return channel != nil && channel.Type == constant.ChannelTypeCodex
}

func detectErrorMessageFromJSONBytes(jsonBytes []byte) string {
if len(jsonBytes) == 0 {
return ""
Expand Down Expand Up @@ -906,10 +902,6 @@ type channelTestSummary struct {
// the system task can surface progress.
func performChannelTests(ctx context.Context, channels []*model.Channel, testUserID int, allowDisable bool, report func(processed, total int)) channelTestSummary {
summary := channelTestSummary{}
var disableThreshold = int64(common.ChannelDisableThreshold * 1000)
if disableThreshold == 0 {
disableThreshold = 10000000 // a impossible value
}

total := len(channels)
for index, channel := range channels {
Expand All @@ -924,7 +916,8 @@ func performChannelTests(ctx context.Context, channels []*model.Channel, testUse
}
isChannelEnabled := channel.Status == common.ChannelStatusEnabled
tik := time.Now()
result := testChannel(ctx, channel, testUserID, "", "", shouldUseStreamForAutomaticChannelTest(channel))
endpointType := channel.EffectiveHealthCheckEndpointType()
result := testChannel(ctx, channel, testUserID, "", endpointType, channel.EffectiveHealthCheckStream())
tok := time.Now()
milliseconds := tok.Sub(tik).Milliseconds()
if ctx != nil && ctx.Err() != nil {
Expand All @@ -942,6 +935,10 @@ func performChannelTests(ctx context.Context, channels []*model.Channel, testUse

// 当错误检查通过，才检查响应时间
if common.AutomaticDisableChannelEnabled && !shouldBanChannel {
disableThreshold := int64(channel.EffectiveHealthCheckDisableThresholdSeconds() * 1000)
if disableThreshold == 0 {
disableThreshold = 10000000 // a impossible value
}
if milliseconds > disableThreshold {
err := fmt.Errorf("响应时间 %.2fs 超过阈值 %.2fs", float64(milliseconds)/1000.0, float64(disableThreshold)/1000.0)
newAPIError = types.NewOpenAIError(err, types.ErrorCodeChannelResponseTimeExceeded, http.StatusRequestTimeout)
Expand All @@ -962,7 +959,7 @@ func performChannelTests(ctx context.Context, channels []*model.Channel, testUse
}

// enable channel
if result.localErr == nil && !isChannelEnabled && service.ShouldEnableChannel(newAPIError, channel.Status) {
if result.localErr == nil && !isChannelEnabled && service.ShouldEnableChannel(newAPIError, channel.Status, channel.EffectiveHealthCheckEnableOnSuccess()) {
service.EnableChannel(channel.Id, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.Name)
summary.Enabled++
}
Expand Down Expand Up @@ -991,9 +988,11 @@ func performChannelTests(ctx context.Context, channels []*model.Channel, testUse
// through here). It honors ctx cancellation so a runner that loses its lease
// stops promptly. mode selects the channel set: an empty mode falls back to the
// configured monitor ChannelTestMode (scheduled behavior), while a manual
// trigger passes ChannelTestModeScheduledAll to test every channel. When notify
// is set the root user is notified on completion. Cross-instance execution is
// guarded by the system task per-type lock, so no process-local guard is needed.
// "test all channels" trigger passes ChannelTestModeScheduledAll and uses the
// same selection rules as scheduled full tests: skip manually disabled channels
// and channels with health_check.enabled=false. When notify is set the root user
// is notified on completion. Cross-instance execution is guarded by the system
// task per-type lock, so no process-local guard is needed.
func runChannelTestTask(ctx context.Context, mode string, notify bool, report func(processed, total int)) (channelTestSummary, error) {
testUserID, err := resolveChannelTestUserID(nil)
if err != nil {
Expand All @@ -1015,12 +1014,19 @@ func runChannelTestTask(ctx context.Context, mode string, notify bool, report fu
return summary, nil
}

// selectChannelsForAutomaticTest chooses channels for both scheduled health
// checks and the manual "test all channels" action. Channels that opt out via
// health_check.enabled=false are always skipped; single-channel manual tests are
// unaffected and still go through TestChannel.
func selectChannelsForAutomaticTest(channels []*model.Channel, mode string) []*model.Channel {
selected := make([]*model.Channel, 0, len(channels))
for _, channel := range channels {
if channel.Status == common.ChannelStatusManuallyDisabled {
continue
}
if !channel.IsAutomaticHealthCheckEnabled() {
continue
}
if mode == operation_setting.ChannelTestModePassiveRecovery && channel.Status != common.ChannelStatusAutoDisabled {
continue
}
Expand All @@ -1030,8 +1036,10 @@ func selectChannelsForAutomaticTest(channels []*model.Channel, mode string) []*m
}

// TestAllChannels enqueues a channel_test system task instead of running the
// test loop inline. If any channel_test task is already active, the manual run is
// rejected so the caller does not mistake a scheduled run for this manual one.
// test loop inline. Selection matches scheduled full tests: manually disabled
// channels and channels with health_check.enabled=false are skipped. If any
// channel_test task is already active, the manual run is rejected so the caller
// does not mistake a scheduled run for this manual one.
func TestAllChannels(c *gin.Context) {
task, created, err := service.EnqueueSystemTask(model.SystemTaskTypeChannelTest, channelTestTaskPayload{
Mode: operation_setting.ChannelTestModeScheduledAll,
Expand Down
87 changes: 87 additions & 0 deletions controller/channel.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2222,3 +2222,90 @@ func OllamaVersion(c *gin.Context) {
},
})
}

const maxChannelHealthCheckBatchSize = 200

type channelHealthCheckBatchItem struct {
Id int `json:"id"`
AutoBan *int `json:"auto_ban,omitempty"`
HealthCheck *dto.ChannelHealthCheckSettings `json:"health_check,omitempty"`
}

type channelHealthCheckBatchRequest struct {
Items []channelHealthCheckBatchItem `json:"items"`
}

type channelHealthCheckBatchItemResult struct {
Id int `json:"id"`
Success bool `json:"success"`
Message string `json:"message,omitempty"`
}

// BatchUpdateChannelHealthCheck narrowly updates settings.health_check and/or
// auto_ban without requiring ChannelSensitiveWrite for the full settings column.
func BatchUpdateChannelHealthCheck(c *gin.Context) {
req := channelHealthCheckBatchRequest{}
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusOK, gin.H{
"success": false,
"message": "参数错误",
})
return
}
if len(req.Items) == 0 {
c.JSON(http.StatusOK, gin.H{
"success": false,
"message": "参数错误",
})
return
}
if len(req.Items) > maxChannelHealthCheckBatchSize {
c.JSON(http.StatusOK, gin.H{
"success": false,
"message": fmt.Sprintf("单次最多更新 %d 个渠道", maxChannelHealthCheckBatchSize),
})
return
}

results := make([]channelHealthCheckBatchItemResult, 0, len(req.Items))
succeeded := 0
for _, item := range req.Items {
result := channelHealthCheckBatchItemResult{Id: item.Id}
if item.Id <= 0 {
result.Message = "invalid channel id"
results = append(results, result)
continue
}
if item.AutoBan == nil && item.HealthCheck == nil {
result.Message = "no health check fields to update"
results = append(results, result)
continue
}
if err := model.UpdateHealthCheckSettings(item.Id, item.AutoBan, item.HealthCheck); err != nil {
result.Message = err.Error()
results = append(results, result)
continue
}
result.Success = true
results = append(results, result)
succeeded++
}

if succeeded > 0 {
model.InitChannelCache()
recordManageAudit(c, "channel.health_check_batch", map[string]interface{}{
"count": len(req.Items),
"succeeded": succeeded,
})
}

c.JSON(http.StatusOK, gin.H{
"success": succeeded == len(req.Items),
"message": "",
"data": gin.H{
"succeeded": succeeded,
"failed": len(req.Items) - succeeded,
"results": results,
},
})
}
30 changes: 30 additions & 0 deletions controller/channel_test_internal_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,36 @@ func TestSelectChannelsForAutomaticTestScheduledSkipsManualDisabled(t *testing.T
require.Equal(t, 2, selected[1].Id)
}

func TestSelectChannelsForAutomaticTestSkipsDisabledHealthCheck(t *testing.T) {
disabled := &model.Channel{Id: 4, Status: common.ChannelStatusEnabled}
disabled.SetOtherSettings(dto.ChannelOtherSettings{
HealthCheck: &dto.ChannelHealthCheckSettings{
Enabled: common.GetPointer(false),
},
})
autoDisabledSkipped := &model.Channel{Id: 5, Status: common.ChannelStatusAutoDisabled}
autoDisabledSkipped.SetOtherSettings(dto.ChannelOtherSettings{
HealthCheck: &dto.ChannelHealthCheckSettings{
Enabled: common.GetPointer(false),
},
})
channels := []*model.Channel{
{Id: 1, Status: common.ChannelStatusEnabled},
{Id: 2, Status: common.ChannelStatusAutoDisabled},
disabled,
autoDisabledSkipped,
}

scheduled := selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModeScheduledAll)
require.Len(t, scheduled, 2)
require.Equal(t, 1, scheduled[0].Id)
require.Equal(t, 2, scheduled[1].Id)

passive := selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModePassiveRecovery)
require.Len(t, passive, 1)
require.Equal(t, 2, passive[0].Id)
}

func TestTestAllChannelsRejectsExistingActiveTask(t *testing.T) {
db := setupModelListControllerTestDB(t)
require.NoError(t, db.AutoMigrate(&model.SystemTask{}, &model.SystemTaskLock{}))
Expand Down
50 changes: 31 additions & 19 deletions dto/channel_settings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,26 +33,38 @@ const (
AwsKeyTypeApiKey AwsKeyType = "api_key"
)

// ChannelHealthCheckSettings stores per-channel automatic health-check overrides.
// Nil pointer fields mean "follow global / existing automatic behavior".
// Enabled defaults to true when HealthCheck is missing or Enabled is nil.
type ChannelHealthCheckSettings struct {
Enabled *bool `json:"enabled,omitempty"`
DisableThresholdSeconds *float64 `json:"disable_threshold_seconds,omitempty"`
EnableOnSuccess *bool `json:"enable_on_success,omitempty"`
EndpointType string `json:"endpoint_type,omitempty"`
Stream *bool `json:"stream,omitempty"`
}

type ChannelOtherSettings struct {
AzureResponsesVersion string `json:"azure_responses_version,omitempty"`
VertexKeyType VertexKeyType `json:"vertex_key_type,omitempty"` // "json" or "api_key"
OpenRouterEnterprise *bool `json:"openrouter_enterprise,omitempty"`
ClaudeBetaQuery bool `json:"claude_beta_query,omitempty"` // Claude 渠道是否强制追加 ?beta=true
AllowServiceTier bool `json:"allow_service_tier,omitempty"` // 是否允许 service_tier 透传（默认过滤以避免额外计费）
AllowInferenceGeo bool `json:"allow_inference_geo,omitempty"` // 是否允许 inference_geo 透传（仅 Claude，默认过滤以满足数据驻留合规
AllowSpeed bool `json:"allow_speed,omitempty"` // 是否允许 speed 透传（仅 Claude，默认过滤以避免意外切换推理速度模式）
AllowSafetyIdentifier bool `json:"allow_safety_identifier,omitempty"` // 是否允许 safety_identifier 透传（默认过滤以保护用户隐私）
DisableStore bool `json:"disable_store,omitempty"` // 是否禁用 store 透传（默认允许透传，禁用后可能导致 Codex 无法使用）
AllowIncludeObfuscation bool `json:"allow_include_obfuscation,omitempty"` // 是否允许 stream_options.include_obfuscation 透传（默认过滤以避免关闭流混淆保护）
DisableTaskPollingSleep bool `json:"disable_task_polling_sleep,omitempty"` // 是否跳过异步任务轮询间隔
AwsKeyType AwsKeyType `json:"aws_key_type,omitempty"`
UpstreamModelUpdateCheckEnabled bool `json:"upstream_model_update_check_enabled,omitempty"` // 是否检测上游模型更新
UpstreamModelUpdateAutoSyncEnabled bool `json:"upstream_model_update_auto_sync_enabled,omitempty"` // 是否自动同步上游模型更新
UpstreamModelUpdateLastCheckTime int64 `json:"upstream_model_update_last_check_time,omitempty"` // 上次检测时间
UpstreamModelUpdateLastDetectedModels []string `json:"upstream_model_update_last_detected_models,omitempty"` // 上次检测到的可加入模型
UpstreamModelUpdateLastRemovedModels []string `json:"upstream_model_update_last_removed_models,omitempty"` // 上次检测到的可删除模型
UpstreamModelUpdateIgnoredModels []string `json:"upstream_model_update_ignored_models,omitempty"` // 手动忽略的模型
AdvancedCustom *AdvancedCustomConfig `json:"advanced_custom,omitempty"`
AzureResponsesVersion string `json:"azure_responses_version,omitempty"`
VertexKeyType VertexKeyType `json:"vertex_key_type,omitempty"` // "json" or "api_key"
OpenRouterEnterprise *bool `json:"openrouter_enterprise,omitempty"`
ClaudeBetaQuery bool `json:"claude_beta_query,omitempty"` // Claude 渠道是否强制追加 ?beta=true
AllowServiceTier bool `json:"allow_service_tier,omitempty"` // 是否允许 service_tier 透传（默认过滤以避免额外计费）
AllowInferenceGeo bool `json:"allow_inference_geo,omitempty"` // 是否允许 inference_geo 透传（仅 Claude，默认过滤以满足数据驻留合规
AllowSpeed bool `json:"allow_speed,omitempty"` // 是否允许 speed 透传（仅 Claude，默认过滤以避免意外切换推理速度模式）
AllowSafetyIdentifier bool `json:"allow_safety_identifier,omitempty"` // 是否允许 safety_identifier 透传（默认过滤以保护用户隐私）
DisableStore bool `json:"disable_store,omitempty"` // 是否禁用 store 透传（默认允许透传，禁用后可能导致 Codex 无法使用）
AllowIncludeObfuscation bool `json:"allow_include_obfuscation,omitempty"` // 是否允许 stream_options.include_obfuscation 透传（默认过滤以避免关闭流混淆保护）
DisableTaskPollingSleep bool `json:"disable_task_polling_sleep,omitempty"` // 是否跳过异步任务轮询间隔
AwsKeyType AwsKeyType `json:"aws_key_type,omitempty"`
UpstreamModelUpdateCheckEnabled bool `json:"upstream_model_update_check_enabled,omitempty"` // 是否检测上游模型更新
UpstreamModelUpdateAutoSyncEnabled bool `json:"upstream_model_update_auto_sync_enabled,omitempty"` // 是否自动同步上游模型更新
UpstreamModelUpdateLastCheckTime int64 `json:"upstream_model_update_last_check_time,omitempty"` // 上次检测时间
UpstreamModelUpdateLastDetectedModels []string `json:"upstream_model_update_last_detected_models,omitempty"` // 上次检测到的可加入模型
UpstreamModelUpdateLastRemovedModels []string `json:"upstream_model_update_last_removed_models,omitempty"` // 上次检测到的可删除模型
UpstreamModelUpdateIgnoredModels []string `json:"upstream_model_update_ignored_models,omitempty"` // 手动忽略的模型
AdvancedCustom *AdvancedCustomConfig `json:"advanced_custom,omitempty"`
HealthCheck *ChannelHealthCheckSettings `json:"health_check,omitempty"`
}

func (s *ChannelOtherSettings) IsOpenRouterEnterprise() bool {
Expand Down
Loading