feat: per-model testing and disable in channel test

controller/channel-test.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -57,7 +58,7 @@ func normalizeChannelTestEndpoint(channel *model.Channel, modelName, endpointTyp
 	return normalized
 }
 
-func testChannel(channel *model.Channel, testModel string, endpointType string, isStream bool) testResult {
+func testChannel(ctx context.Context, channel *model.Channel, testModel string, endpointType string, isStream bool) testResult {
 	tik := time.Now()
 	var unsupportedTestChannelTypes = []int{
 		constant.ChannelTypeMidjourney,
@@ -138,10 +139,11 @@ func testChannel(channel *model.Channel, testModel string, endpointType string,
 
 	c.Request = &http.Request{
 		Method: "POST",
-		URL:    &url.URL{Path: requestPath}, // 使用动态路径
+		URL:    &url.URL{Path: requestPath},
 		Body:   nil,
 		Header: make(http.Header),
 	}
+	c.Request = c.Request.WithContext(ctx)
 
 	cache, err := model.GetUserCache(1)
 	if err != nil {
@@ -835,7 +837,7 @@ func TestChannel(c *gin.Context) {
 	endpointType := c.Query("endpoint_type")
 	isStream, _ := strconv.ParseBool(c.Query("stream"))
 	tik := time.Now()
-	result := testChannel(channel, testModel, endpointType, isStream)
+	result := testChannel(c.Request.Context(), channel, testModel, endpointType, isStream)
 	if result.localErr != nil {
 		resp := gin.H{
 			"success": false,
@@ -901,41 +903,133 @@ func testAllChannels(notify bool) error {
 				continue
 			}
 			isChannelEnabled := channel.Status == common.ChannelStatusEnabled
-			tik := time.Now()
-			result := testChannel(channel, "", "", shouldUseStreamForAutomaticChannelTest(channel))
-			tok := time.Now()
-			milliseconds := tok.Sub(tik).Milliseconds()
-
-			shouldBanChannel := false
-			newAPIError := result.newAPIError
-			// request error disables the channel
-			if newAPIError != nil {
-				shouldBanChannel = service.ShouldDisableChannel(result.newAPIError)
+
+			models := channel.GetModels()
+			if len(models) == 0 {
+				continue
 			}
 
-			// 当错误检查通过，才检查响应时间
-			if common.AutomaticDisableChannelEnabled && !shouldBanChannel {
-				if milliseconds > disableThreshold {
-					err := fmt.Errorf("响应时间 %.2fs 超过阈值 %.2fs", float64(milliseconds)/1000.0, float64(disableThreshold)/1000.0)
-					newAPIError = types.NewOpenAIError(err, types.ErrorCodeChannelResponseTimeExceeded, http.StatusRequestTimeout)
-					shouldBanChannel = true
+			var totalMs int64
+			var testedCount int64
+			channelLevelErrorOccurred := false
+
+			for _, testModelName := range models {
+				testModelName = strings.TrimSpace(testModelName)
+				if testModelName == "" {
+					continue
+				}
+
+				lowerModelName := strings.ToLower(testModelName)
+				if strings.Contains(lowerModelName, "seedream") ||
+					strings.Contains(lowerModelName, "image-preview") {
+					model.RecordChannelTestHistory(channel.Id, channel.Name, testModelName, "unsupported", 0, "image generation model test is not supported")
+					continue
+				}
+
+				testTimeout := 120 * time.Second
+				tik := time.Now()
+				ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+				resultCh := make(chan testResult, 1)
+				go func() {
+					resultCh <- testChannel(ctx, channel, testModelName, "", false)
+				}()
+				var result testResult
+				select {
+				case result = <-resultCh:
+				case <-ctx.Done():
+					result = testResult{
+						localErr:    fmt.Errorf("测试超时（%ds），模型「%s」未在限定时间内响应", int(testTimeout.Seconds()), testModelName),
+						newAPIError: types.NewOpenAIError(fmt.Errorf("test timeout after %ds", int(testTimeout.Seconds())), types.ErrorCodeChannelResponseTimeExceeded, http.StatusRequestTimeout),
+					}
+				}
+				cancel()
+				tok := time.Now()
+				milliseconds := tok.Sub(tik).Milliseconds()
+				totalMs += milliseconds
+				testedCount++
+
+				shouldBanModel := false
+				newAPIError := result.newAPIError
+				errMsg := ""
+				testStatus := "operational"
+
+				if result.localErr != nil {
+					errMsg = result.localErr.Error()
+					if strings.Contains(errMsg, "not supported") ||
+						strings.Contains(errMsg, "invalid image request type") ||
+						strings.Contains(errMsg, "invalid embedding request type") ||
+						strings.Contains(errMsg, "invalid rerank request type") {
+						testStatus = "unsupported"
+					}
+				}
+
+				if newAPIError != nil {
+					shouldBanModel = service.ShouldDisableChannel(newAPIError)
+
+					if shouldBanModel && service.IsChannelLevelError(newAPIError) {
+						if isChannelEnabled && channel.GetAutoBan() && result.context != nil {
+							processChannelError(result.context, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError)
+						}
+						channelLevelErrorOccurred = true
+						testStatus = "failed"
+						model.RecordChannelTestHistory(channel.Id, channel.Name, testModelName, testStatus, milliseconds, errMsg)
+						break
+					}
+				}
+
+				// 检查响应时间是否超阈值
+				if common.AutomaticDisableChannelEnabled && !shouldBanModel {
+					if milliseconds > disableThreshold {
+						err := fmt.Errorf("响应时间 %.2fs 超过阈值 %.2fs", float64(milliseconds)/1000.0, float64(disableThreshold)/1000.0)
+						newAPIError = types.NewOpenAIError(err, types.ErrorCodeChannelResponseTimeExceeded, http.StatusRequestTimeout)
+						shouldBanModel = true
+						testStatus = "timeout"
+						errMsg = err.Error()
+					}
 				}
-			}
 
-			// disable channel
-			if isChannelEnabled && shouldBanChannel && channel.GetAutoBan() {
-				processChannelError(result.context, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError)
+				if common.AutomaticDisableChannelEnabled && isChannelEnabled && shouldBanModel && channel.GetAutoBan() && testStatus != "unsupported" {
+					reason := "测试失败"
+					if newAPIError != nil {
+						reason = newAPIError.ErrorWithStatusCode()
+					}
+					service.DisableChannelModel(channel.Id, channel.Name, testModelName, reason)
+					testStatus = "failed"
+				}
+
+				if common.AutomaticEnableChannelEnabled && newAPIError == nil && testStatus == "operational" {
+					if !model.IsAbilityModelEnabled(channel.Id, testModelName) {
+						service.EnableChannelModel(channel.Id, channel.Name, testModelName)
+					}
+				}
+
+				model.RecordChannelTestHistory(channel.Id, channel.Name, testModelName, testStatus, milliseconds, errMsg)
+				time.Sleep(common.RequestInterval)
 			}
 
-			// enable channel
-			if !isChannelEnabled && service.ShouldEnableChannel(newAPIError, channel.Status) {
-				service.EnableChannel(channel.Id, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.Name)
+			// 通道级：如果未发生通道级错误，且通道之前被自动禁用，且本次所有模型都测试通过，则重新启用通道
+			if common.AutomaticEnableChannelEnabled && !channelLevelErrorOccurred && !isChannelEnabled && channel.Status == common.ChannelStatusAutoDisabled {
+				// 检查是否所有模型测试都成功（通过检查是否有任何模型被禁用）
+				allModelsOk := true
+				for _, m := range models {
+					if !model.IsAbilityModelEnabled(channel.Id, strings.TrimSpace(m)) {
+						allModelsOk = false
+						break
+					}
+				}
+				if allModelsOk {
+					service.EnableChannel(channel.Id, "", channel.Name)
+				}
 			}
 
-			channel.UpdateResponseTime(milliseconds)
-			time.Sleep(common.RequestInterval)
+			// 使用所有模型的平均响应时间更新通道响应时间
+			if testedCount > 0 {
+				channel.UpdateResponseTime(totalMs / testedCount)
+			}
 		}
 
+		model.PruneChannelTestHistory(30)
+
 		if notify {
 			service.NotifyRootUser(dto.NotifyTypeChannelTest, "通道测试完成", "所有通道测试已完成")
 		}

model/ability.go

@@ -264,6 +264,21 @@ func UpdateAbilityStatus(channelId int, status bool) error {
 	return DB.Model(&Ability{}).Where("channel_id = ?", channelId).Select("enabled").Update("enabled", status).Error
 }
 
+// UpdateAbilityModelStatus updates the enabled status of a specific model within a channel
+func UpdateAbilityModelStatus(channelId int, modelName string, status bool) error {
+	err := DB.Model(&Ability{}).Where("channel_id = ? AND model = ?", channelId, modelName).Select("enabled").Update("enabled", status).Error
+	if err == nil {
+		InitChannelCache()
+	}
+	return err
+}
+
+func IsAbilityModelEnabled(channelId int, modelName string) bool {
+	var count int64
+	DB.Model(&Ability{}).Where("channel_id = ? AND model = ? AND enabled = ?", channelId, modelName, true).Count(&count)
+	return count > 0
+}
+
 func UpdateAbilityStatusByTag(tag string, status bool) error {
 	return DB.Model(&Ability{}).Where("tag = ?", tag).Select("enabled").Update("enabled", status).Error
 }

model/channel_cache.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"math/rand"
 	"sort"
-	"strings"
 	"sync"
 	"time"
 
@@ -30,28 +29,25 @@ func InitChannelCache() {
 	}
 	var abilities []*Ability
 	DB.Find(&abilities)
-	groups := make(map[string]bool)
-	for _, ability := range abilities {
-		groups[ability.Group] = true
-	}
 	newGroup2model2channels := make(map[string]map[string][]int)
-	for group := range groups {
-		newGroup2model2channels[group] = make(map[string][]int)
-	}
-	for _, channel := range channels {
-		if channel.Status != common.ChannelStatusEnabled {
-			continue // skip disabled channels
+	for _, ability := range abilities {
+		if !ability.Enabled {
+			continue
 		}
-		groups := strings.Split(channel.Group, ",")
-		for _, group := range groups {
-			models := strings.Split(channel.Models, ",")
-			for _, model := range models {
-				if _, ok := newGroup2model2channels[group][model]; !ok {
-					newGroup2model2channels[group][model] = make([]int, 0)
-				}
-				newGroup2model2channels[group][model] = append(newGroup2model2channels[group][model], channel.Id)
-			}
+		channel, ok := newChannelId2channel[ability.ChannelId]
+		if !ok || channel.Status != common.ChannelStatusEnabled {
+			continue
+		}
+		if _, ok := newGroup2model2channels[ability.Group]; !ok {
+			newGroup2model2channels[ability.Group] = make(map[string][]int)
+		}
+		if _, ok := newGroup2model2channels[ability.Group][ability.Model]; !ok {
+			newGroup2model2channels[ability.Group][ability.Model] = make([]int, 0)
 		}
+		newGroup2model2channels[ability.Group][ability.Model] = append(
+			newGroup2model2channels[ability.Group][ability.Model],
+			ability.ChannelId,
+		)
 	}
 
 	// sort by priority

model/channel_test_history.go

@@ -0,0 +1,44 @@
+package model
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/bytedance/gopkg/util/gopool"
+)
+
+// ChannelTestHistory records per-model test results for availability tracking
+type ChannelTestHistory struct {
+	Id           int       `json:"id" gorm:"primaryKey;autoIncrement"`
+	ChannelId    int       `json:"channel_id" gorm:"index"`
+	ChannelName  string    `json:"channel_name"`
+	TestModel    string    `json:"test_model"`
+	Status       string    `json:"status"`        // operational, failed, timeout, unsupported
+	ResponseTime int64     `json:"response_time"` // ms
+	ErrorMessage string    `json:"error_message" gorm:"type:text"`
+	TestedAt     time.Time `json:"tested_at" gorm:"index"`
+}
+
+func RecordChannelTestHistory(channelId int, channelName, testModel, status string, responseTime int64, errMsg string) {
+	gopool.Go(func() {
+		history := ChannelTestHistory{
+			ChannelId:    channelId,
+			ChannelName:  channelName,
+			TestModel:    testModel,
+			Status:       status,
+			ResponseTime: responseTime,
+			ErrorMessage: errMsg,
+			TestedAt:     time.Now(),
+		}
+		if err := DB.Create(&history).Error; err != nil {
+			common.SysError(fmt.Sprintf("failed to record channel test history: %v", err))
+		}
+	})
+}
+
+func PruneChannelTestHistory(retentionDays int) int64 {
+	cutoff := time.Now().AddDate(0, 0, -retentionDays)
+	result := DB.Where("tested_at < ?", cutoff).Delete(&ChannelTestHistory{})
+	return result.RowsAffected
+}

model/main.go

@@ -281,6 +281,7 @@ func migrateDB() error {
 		&CustomOAuthProvider{},
 		&UserOAuthBinding{},
 		&PerfMetric{},
+		&ChannelTestHistory{},
 	)
 	if err != nil {
 		return err
@@ -330,6 +331,7 @@ func migrateDBFast() error {
 		{&CustomOAuthProvider{}, "CustomOAuthProvider"},
 		{&UserOAuthBinding{}, "UserOAuthBinding"},
 		{&PerfMetric{}, "PerfMetric"},
+		{&ChannelTestHistory{}, "ChannelTestHistory"},
 	}
 	// 动态计算migration数量，确保errChan缓冲区足够大
 	errChan := make(chan error, len(migrations))