From 8b020a82f96cf8cc0bb8951fa0499853796e105c Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Apr 2026 22:59:08 +0800 Subject: [PATCH 01/16] feat: add governor settings and ordered key helpers --- dto/channel_settings.go | 15 ++++ dto/channel_settings_test.go | 53 ++++++++++++ model/channel.go | 151 +++++++++++++++++++++------------ model/channel_governor_test.go | 43 ++++++++++ 4 files changed, 207 insertions(+), 55 deletions(-) create mode 100644 dto/channel_settings_test.go create mode 100644 model/channel_governor_test.go diff --git a/dto/channel_settings.go b/dto/channel_settings.go index 8d7466d25966..e3ef3b00ea55 100644 --- a/dto/channel_settings.go +++ b/dto/channel_settings.go @@ -7,6 +7,21 @@ type ChannelSettings struct { PassThroughBodyEnabled bool `json:"pass_through_body_enabled,omitempty"` SystemPrompt string `json:"system_prompt,omitempty"` SystemPromptOverride bool `json:"system_prompt_override,omitempty"` + Governor *GovernorSettings `json:"governor,omitempty"` +} + +type GovernorSettings struct { + Enabled bool `json:"enabled"` + ChannelMaxRPM int `json:"channel_max_rpm"` + ChannelCooldownSeconds int `json:"channel_cooldown_seconds"` + ChannelCooldownOnStatuses []int `json:"channel_cooldown_on_statuses"` + KeyMaxConcurrency int `json:"key_max_concurrency"` + KeyCooldownSeconds int `json:"key_cooldown_seconds"` + KeyCooldownOnStatuses []int `json:"key_cooldown_on_statuses"` + ReservationLeaseSeconds int `json:"reservation_lease_seconds"` + ReservationHeartbeatSeconds int `json:"reservation_heartbeat_seconds"` + ShortWaitMs int `json:"short_wait_ms"` + RespectRetryAfter bool `json:"respect_retry_after"` } type VertexKeyType string diff --git a/dto/channel_settings_test.go b/dto/channel_settings_test.go new file mode 100644 index 000000000000..2c76d719c409 --- /dev/null +++ b/dto/channel_settings_test.go @@ -0,0 +1,53 @@ +package dto + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/QuantumNous/new-api/common" +) + +func TestChannelSettings_GovernorRoundTrip(t *testing.T) { + raw := ` +{ + "proxy": "https://example.com/proxy", + "governor": { + "enabled": true, + "channel_max_rpm": 120, + "channel_cooldown_seconds": 30, + "channel_cooldown_on_statuses": [2, 3], + "key_max_concurrency": 5, + "key_cooldown_seconds": 10, + "key_cooldown_on_statuses": [2], + "reservation_lease_seconds": 60, + "reservation_heartbeat_seconds": 15, + "short_wait_ms": 200, + "respect_retry_after": true + } +} +` + + var settings ChannelSettings + require.NoError(t, common.Unmarshal([]byte(raw), &settings)) + require.NotNil(t, settings.Governor) + require.True(t, settings.Governor.Enabled) + require.Equal(t, 120, settings.Governor.ChannelMaxRPM) + require.Equal(t, 30, settings.Governor.ChannelCooldownSeconds) + require.Equal(t, []int{2, 3}, settings.Governor.ChannelCooldownOnStatuses) + require.Equal(t, 5, settings.Governor.KeyMaxConcurrency) + require.Equal(t, 10, settings.Governor.KeyCooldownSeconds) + require.Equal(t, []int{2}, settings.Governor.KeyCooldownOnStatuses) + require.Equal(t, 60, settings.Governor.ReservationLeaseSeconds) + require.Equal(t, 15, settings.Governor.ReservationHeartbeatSeconds) + require.Equal(t, 200, settings.Governor.ShortWaitMs) + require.True(t, settings.Governor.RespectRetryAfter) + + marshaled, err := common.Marshal(settings) + require.NoError(t, err) + + var roundTrip ChannelSettings + require.NoError(t, common.Unmarshal(marshaled, &roundTrip)) + require.NotNil(t, roundTrip.Governor) + require.Equal(t, settings.Governor, roundTrip.Governor) +} diff --git a/model/channel.go b/model/channel.go index f256b54ce35b..1d193522e732 100644 --- a/model/channel.go +++ b/model/channel.go @@ -102,91 +102,132 @@ func (channel *Channel) GetKeys() []string { return keys } -func (channel *Channel) GetNextEnabledKey() (string, int, *types.NewAPIError) { - // If not in multi-key mode, return the original key string directly. - if !channel.ChannelInfo.IsMultiKey { - return channel.Key, 0, nil - } - - // Obtain all keys (split by \n) +func (channel *Channel) OrderedEnabledKeyIndices() ([]int, *types.NewAPIError) { keys := channel.GetKeys() if len(keys) == 0 { - // No keys available, return error, should disable the channel - return "", 0, types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) + return nil, types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) } - lock := GetChannelPollingLock(channel.Id) - lock.Lock() - defer lock.Unlock() - - statusList := channel.ChannelInfo.MultiKeyStatusList - // helper to get key status, default to enabled when missing getStatus := func(idx int) int { - if statusList == nil { + if channel.ChannelInfo.MultiKeyStatusList == nil { return common.ChannelStatusEnabled } - if status, ok := statusList[idx]; ok { + if status, ok := channel.ChannelInfo.MultiKeyStatusList[idx]; ok { return status } return common.ChannelStatusEnabled } - // Collect indexes of enabled keys - enabledIdx := make([]int, 0, len(keys)) + enabled := make([]int, 0, len(keys)) for i := range keys { if getStatus(i) == common.ChannelStatusEnabled { - enabledIdx = append(enabledIdx, i) + enabled = append(enabled, i) } } - // If no specific status list or none enabled, return an explicit error so caller can - // properly handle a channel with no available keys (e.g. mark channel disabled). - // Returning the first key here caused requests to keep using an already-disabled key. - if len(enabledIdx) == 0 { - return "", 0, types.NewError(errors.New("no enabled keys"), types.ErrorCodeChannelNoAvailableKey) + if len(enabled) == 0 { + return nil, types.NewError(errors.New("no enabled keys"), types.ErrorCodeChannelNoAvailableKey) } - switch channel.ChannelInfo.MultiKeyMode { - case constant.MultiKeyModeRandom: - // Randomly pick one enabled key - selectedIdx := enabledIdx[rand.Intn(len(enabledIdx))] - return keys[selectedIdx], selectedIdx, nil - case constant.MultiKeyModePolling: - // Use channel-specific lock to ensure thread-safe polling + if channel.ChannelInfo.IsMultiKey && channel.ChannelInfo.MultiKeyMode == constant.MultiKeyModePolling { + start := channel.ChannelInfo.MultiKeyPollingIndex + if start < 0 || start >= len(keys) { + start = 0 + } + ordered := make([]int, 0, len(enabled)) + for i := 0; i < len(keys); i++ { + idx := (start + i) % len(keys) + if getStatus(idx) == common.ChannelStatusEnabled { + ordered = append(ordered, idx) + } + } + return ordered, nil + } + + return enabled, nil +} + +func (channel *Channel) KeyAt(index int) (string, *types.NewAPIError) { + keys := channel.GetKeys() + if len(keys) == 0 { + return "", types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) + } + if index < 0 || index >= len(keys) { + return "", types.NewError(errors.New("invalid key index"), types.ErrorCodeChannelInvalidKey) + } + return keys[index], nil +} +func (channel *Channel) CommitSelectedKeyIndex(index int) *types.NewAPIError { + keys := channel.GetKeys() + if len(keys) == 0 { + return types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) + } + if index < 0 || index >= len(keys) { + return types.NewError(errors.New("invalid key index"), types.ErrorCodeChannelInvalidKey) + } + if !channel.ChannelInfo.IsMultiKey || channel.ChannelInfo.MultiKeyMode != constant.MultiKeyModePolling { + return nil + } + + channel.ChannelInfo.MultiKeyPollingIndex = (index + 1) % len(keys) + if common.DebugEnabled { + println(fmt.Sprintf("channel %d polling index: %d", channel.Id, channel.ChannelInfo.MultiKeyPollingIndex)) + } + if !common.MemoryCacheEnabled { + _ = channel.SaveChannelInfo() + } + return nil +} + +func (channel *Channel) GetNextEnabledKey() (string, int, *types.NewAPIError) { + if !channel.ChannelInfo.IsMultiKey { + return channel.Key, 0, nil + } + + keys := channel.GetKeys() + if len(keys) == 0 { + return "", 0, types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) + } + + lock := GetChannelPollingLock(channel.Id) + lock.Lock() + defer lock.Unlock() + + if channel.ChannelInfo.MultiKeyMode == constant.MultiKeyModePolling { channelInfo, err := CacheGetChannelInfo(channel.Id) if err != nil { return "", 0, types.NewError(err, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) } - //println("before polling index:", channel.ChannelInfo.MultiKeyPollingIndex) - defer func() { - if common.DebugEnabled { - println(fmt.Sprintf("channel %d polling index: %d", channel.Id, channel.ChannelInfo.MultiKeyPollingIndex)) - } - if !common.MemoryCacheEnabled { - _ = channel.SaveChannelInfo() - } else { - // CacheUpdateChannel(channel) - } - }() - // Start from the saved polling index and look for the next enabled key start := channelInfo.MultiKeyPollingIndex if start < 0 || start >= len(keys) { start = 0 } - for i := 0; i < len(keys); i++ { - idx := (start + i) % len(keys) - if getStatus(idx) == common.ChannelStatusEnabled { - // update polling index for next call (point to the next position) - channel.ChannelInfo.MultiKeyPollingIndex = (idx + 1) % len(keys) - return keys[idx], idx, nil - } + channel.ChannelInfo.MultiKeyPollingIndex = start + } + + enabledIdx, err := channel.OrderedEnabledKeyIndices() + if err != nil { + return "", 0, err + } + + var selectedIdx int + switch channel.ChannelInfo.MultiKeyMode { + case constant.MultiKeyModeRandom: + selectedIdx = enabledIdx[rand.Intn(len(enabledIdx))] + case constant.MultiKeyModePolling: + selectedIdx = enabledIdx[0] + if commitErr := channel.CommitSelectedKeyIndex(selectedIdx); commitErr != nil { + return "", 0, commitErr } - // Fallback – should not happen, but return first enabled key - return keys[enabledIdx[0]], enabledIdx[0], nil default: - // Unknown mode, default to first enabled key (or original key string) - return keys[enabledIdx[0]], enabledIdx[0], nil + selectedIdx = enabledIdx[0] + } + + selectedKey, keyErr := channel.KeyAt(selectedIdx) + if keyErr != nil { + return "", 0, keyErr } + return selectedKey, selectedIdx, nil } func (channel *Channel) SaveChannelInfo() error { diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go new file mode 100644 index 000000000000..1e8fee24861d --- /dev/null +++ b/model/channel_governor_test.go @@ -0,0 +1,43 @@ +package model + +import ( + "testing" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/stretchr/testify/require" +) + +func TestChannelOrderedEnabledKeyIndices_PollingDoesNotAdvanceUntilCommitted(t *testing.T) { + channel := &Channel{ + Id: 42, + Key: "k0\nk1\nk2", + ChannelInfo: ChannelInfo{ + IsMultiKey: true, + MultiKeySize: 3, + MultiKeyMode: constant.MultiKeyModePolling, + MultiKeyPollingIndex: 1, + MultiKeyStatusList: map[int]int{ + 2: common.ChannelStatusAutoDisabled, + }, + }, + } + + lock := GetChannelPollingLock(channel.Id) + lock.Lock() + ordered, err := channel.OrderedEnabledKeyIndices() + lock.Unlock() + require.NoError(t, err) + require.Equal(t, []int{1, 0}, ordered) + require.Equal(t, 1, channel.ChannelInfo.MultiKeyPollingIndex) + + key, err := channel.KeyAt(0) + require.NoError(t, err) + require.Equal(t, "k0", key) + + lock.Lock() + err = channel.CommitSelectedKeyIndex(1) + lock.Unlock() + require.NoError(t, err) + require.Equal(t, 2, channel.ChannelInfo.MultiKeyPollingIndex) +} From 45374e89a0d6dc538d05445470c40a16ba28e23c Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Apr 2026 23:37:21 +0800 Subject: [PATCH 02/16] fix: align governor helpers with cache --- model/channel.go | 27 +++--- model/channel_governor_test.go | 158 ++++++++++++++++++++++++++++----- 2 files changed, 150 insertions(+), 35 deletions(-) diff --git a/model/channel.go b/model/channel.go index 1d193522e732..a2b42d0fdb77 100644 --- a/model/channel.go +++ b/model/channel.go @@ -128,8 +128,13 @@ func (channel *Channel) OrderedEnabledKeyIndices() ([]int, *types.NewAPIError) { return nil, types.NewError(errors.New("no enabled keys"), types.ErrorCodeChannelNoAvailableKey) } + var start int if channel.ChannelInfo.IsMultiKey && channel.ChannelInfo.MultiKeyMode == constant.MultiKeyModePolling { - start := channel.ChannelInfo.MultiKeyPollingIndex + channelInfo, err := CacheGetChannelInfo(channel.Id) + if err != nil { + return nil, types.NewError(err, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) + } + start = channelInfo.MultiKeyPollingIndex if start < 0 || start >= len(keys) { start = 0 } @@ -169,7 +174,13 @@ func (channel *Channel) CommitSelectedKeyIndex(index int) *types.NewAPIError { return nil } - channel.ChannelInfo.MultiKeyPollingIndex = (index + 1) % len(keys) + channelInfo, err := CacheGetChannelInfo(channel.Id) + if err != nil { + return types.NewError(err, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) + } + + channelInfo.MultiKeyPollingIndex = (index + 1) % len(keys) + channel.ChannelInfo.MultiKeyPollingIndex = channelInfo.MultiKeyPollingIndex if common.DebugEnabled { println(fmt.Sprintf("channel %d polling index: %d", channel.Id, channel.ChannelInfo.MultiKeyPollingIndex)) } @@ -193,18 +204,6 @@ func (channel *Channel) GetNextEnabledKey() (string, int, *types.NewAPIError) { lock.Lock() defer lock.Unlock() - if channel.ChannelInfo.MultiKeyMode == constant.MultiKeyModePolling { - channelInfo, err := CacheGetChannelInfo(channel.Id) - if err != nil { - return "", 0, types.NewError(err, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) - } - start := channelInfo.MultiKeyPollingIndex - if start < 0 || start >= len(keys) { - start = 0 - } - channel.ChannelInfo.MultiKeyPollingIndex = start - } - enabledIdx, err := channel.OrderedEnabledKeyIndices() if err != nil { return "", 0, err diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go index 1e8fee24861d..22863c814966 100644 --- a/model/channel_governor_test.go +++ b/model/channel_governor_test.go @@ -9,35 +9,151 @@ import ( ) func TestChannelOrderedEnabledKeyIndices_PollingDoesNotAdvanceUntilCommitted(t *testing.T) { + withMemoryCache(t, func() { + channel := &Channel{ + Id: 42, + Key: "k0\nk1\nk2", + ChannelInfo: ChannelInfo{ + IsMultiKey: true, + MultiKeySize: 3, + MultiKeyMode: constant.MultiKeyModePolling, + MultiKeyPollingIndex: 1, + MultiKeyStatusList: map[int]int{ + 2: common.ChannelStatusAutoDisabled, + }, + }, + } + + registerCachedChannel(channel) + + lock := GetChannelPollingLock(channel.Id) + lock.Lock() + ordered, err := channel.OrderedEnabledKeyIndices() + lock.Unlock() + require.NoError(t, err) + require.Equal(t, []int{1, 0}, ordered) + require.Equal(t, 1, channel.ChannelInfo.MultiKeyPollingIndex) + + lock.Lock() + require.NoError(t, channel.CommitSelectedKeyIndex(ordered[0])) + lock.Unlock() + require.Equal(t, 2, channel.ChannelInfo.MultiKeyPollingIndex) + + channelSyncLock.RLock() + require.Equal(t, 2, channelsIDM[channel.Id].ChannelInfo.MultiKeyPollingIndex) + channelSyncLock.RUnlock() + + stale := *channel + stale.ChannelInfo.MultiKeyPollingIndex = 0 + + staleLock := GetChannelPollingLock(stale.Id) + staleLock.Lock() + staleOrdered, err := stale.OrderedEnabledKeyIndices() + staleLock.Unlock() + require.NoError(t, err) + require.Equal(t, []int{0, 1}, staleOrdered) + + staleLock.Lock() + require.NoError(t, stale.CommitSelectedKeyIndex(staleOrdered[0])) + staleLock.Unlock() + + channelSyncLock.RLock() + require.Equal(t, 1, channelsIDM[channel.Id].ChannelInfo.MultiKeyPollingIndex) + channelSyncLock.RUnlock() + require.Equal(t, 1, stale.ChannelInfo.MultiKeyPollingIndex) + + key, err := stale.KeyAt(staleOrdered[0]) + require.NoError(t, err) + require.Equal(t, "k0", key) + }) +} + +func TestChannelGetNextEnabledKey_PollingUsesCanonicalState(t *testing.T) { + withMemoryCache(t, func() { + channel := &Channel{ + Id: 43, + Key: "k0\nk1\nk2", + ChannelInfo: ChannelInfo{ + IsMultiKey: true, + MultiKeySize: 3, + MultiKeyMode: constant.MultiKeyModePolling, + MultiKeyPollingIndex: 2, + MultiKeyStatusList: map[int]int{ + 2: common.ChannelStatusAutoDisabled, + }, + }, + } + + registerCachedChannel(channel) + + key, idx, err := channel.GetNextEnabledKey() + require.NoError(t, err) + require.Equal(t, "k0", key) + require.Equal(t, 0, idx) + require.Equal(t, 1, channel.ChannelInfo.MultiKeyPollingIndex) + + key2, idx2, err := channel.GetNextEnabledKey() + require.NoError(t, err) + require.Equal(t, "k1", key2) + require.Equal(t, 1, idx2) + require.Equal(t, 2, channel.ChannelInfo.MultiKeyPollingIndex) + }) +} + +func TestChannelGetNextEnabledKey_RandomReturnsSingleKey(t *testing.T) { channel := &Channel{ - Id: 42, - Key: "k0\nk1\nk2", + Id: 44, + Key: "solo", ChannelInfo: ChannelInfo{ IsMultiKey: true, - MultiKeySize: 3, - MultiKeyMode: constant.MultiKeyModePolling, - MultiKeyPollingIndex: 1, - MultiKeyStatusList: map[int]int{ - 2: common.ChannelStatusAutoDisabled, - }, + MultiKeySize: 1, + MultiKeyMode: constant.MultiKeyModeRandom, }, } - lock := GetChannelPollingLock(channel.Id) - lock.Lock() - ordered, err := channel.OrderedEnabledKeyIndices() - lock.Unlock() + key, idx, err := channel.GetNextEnabledKey() require.NoError(t, err) - require.Equal(t, []int{1, 0}, ordered) - require.Equal(t, 1, channel.ChannelInfo.MultiKeyPollingIndex) + require.Equal(t, "solo", key) + require.Equal(t, 0, idx) +} - key, err := channel.KeyAt(0) - require.NoError(t, err) - require.Equal(t, "k0", key) +func TestChannelGetNextEnabledKey_ReturnsKeyWhenNotMultiKey(t *testing.T) { + channel := &Channel{ + Key: "single", + } - lock.Lock() - err = channel.CommitSelectedKeyIndex(1) - lock.Unlock() + key, idx, err := channel.GetNextEnabledKey() require.NoError(t, err) - require.Equal(t, 2, channel.ChannelInfo.MultiKeyPollingIndex) + require.Equal(t, "single", key) + require.Equal(t, 0, idx) +} + +func withMemoryCache(t *testing.T, run func()) { + original := common.MemoryCacheEnabled + common.MemoryCacheEnabled = true + defer func() { + common.MemoryCacheEnabled = original + }() + + channelSyncLock.Lock() + previous := channelsIDM + channelsIDM = map[int]*Channel{} + channelSyncLock.Unlock() + + defer func() { + channelSyncLock.Lock() + channelsIDM = previous + channelSyncLock.Unlock() + }() + + run() +} + +func registerCachedChannel(channel *Channel) { + channelSyncLock.Lock() + if channelsIDM == nil { + channelsIDM = map[int]*Channel{} + } + channelsIDM[channel.Id] = channel + channelSyncLock.Unlock() } From c596e20f68458cd1256037c15d7d055ee82a6e6d Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 00:01:10 +0800 Subject: [PATCH 03/16] test: tighten random-mode coverage --- model/channel_governor_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go index 22863c814966..431150fc6f6a 100644 --- a/model/channel_governor_test.go +++ b/model/channel_governor_test.go @@ -100,21 +100,27 @@ func TestChannelGetNextEnabledKey_PollingUsesCanonicalState(t *testing.T) { }) } -func TestChannelGetNextEnabledKey_RandomReturnsSingleKey(t *testing.T) { +func TestChannelGetNextEnabledKey_RandomUsesEnabledCandidates(t *testing.T) { channel := &Channel{ Id: 44, - Key: "solo", + Key: "k0\nk1\nk2", ChannelInfo: ChannelInfo{ IsMultiKey: true, - MultiKeySize: 1, + MultiKeySize: 3, MultiKeyMode: constant.MultiKeyModeRandom, + MultiKeyPollingIndex: 5, + MultiKeyStatusList: map[int]int{ + 1: common.ChannelStatusAutoDisabled, + }, }, } + allowed := map[int]string{0: "k0", 2: "k2"} key, idx, err := channel.GetNextEnabledKey() require.NoError(t, err) - require.Equal(t, "solo", key) - require.Equal(t, 0, idx) + require.Contains(t, allowed, idx) + require.Equal(t, allowed[idx], key) + require.Equal(t, 5, channel.ChannelInfo.MultiKeyPollingIndex) } func TestChannelGetNextEnabledKey_ReturnsKeyWhenNotMultiKey(t *testing.T) { From 9d3834de553b6f33ac3b77529e61deb6225f48f2 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 00:23:58 +0800 Subject: [PATCH 04/16] test: nil-check governor helper errors --- model/channel_governor_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go index 431150fc6f6a..30dcb5cf3401 100644 --- a/model/channel_governor_test.go +++ b/model/channel_governor_test.go @@ -30,12 +30,12 @@ func TestChannelOrderedEnabledKeyIndices_PollingDoesNotAdvanceUntilCommitted(t * lock.Lock() ordered, err := channel.OrderedEnabledKeyIndices() lock.Unlock() - require.NoError(t, err) + require.Nil(t, err) require.Equal(t, []int{1, 0}, ordered) require.Equal(t, 1, channel.ChannelInfo.MultiKeyPollingIndex) lock.Lock() - require.NoError(t, channel.CommitSelectedKeyIndex(ordered[0])) + require.Nil(t, channel.CommitSelectedKeyIndex(ordered[0])) lock.Unlock() require.Equal(t, 2, channel.ChannelInfo.MultiKeyPollingIndex) @@ -50,11 +50,11 @@ func TestChannelOrderedEnabledKeyIndices_PollingDoesNotAdvanceUntilCommitted(t * staleLock.Lock() staleOrdered, err := stale.OrderedEnabledKeyIndices() staleLock.Unlock() - require.NoError(t, err) + require.Nil(t, err) require.Equal(t, []int{0, 1}, staleOrdered) staleLock.Lock() - require.NoError(t, stale.CommitSelectedKeyIndex(staleOrdered[0])) + require.Nil(t, stale.CommitSelectedKeyIndex(staleOrdered[0])) staleLock.Unlock() channelSyncLock.RLock() @@ -63,7 +63,7 @@ func TestChannelOrderedEnabledKeyIndices_PollingDoesNotAdvanceUntilCommitted(t * require.Equal(t, 1, stale.ChannelInfo.MultiKeyPollingIndex) key, err := stale.KeyAt(staleOrdered[0]) - require.NoError(t, err) + require.Nil(t, err) require.Equal(t, "k0", key) }) } @@ -87,13 +87,13 @@ func TestChannelGetNextEnabledKey_PollingUsesCanonicalState(t *testing.T) { registerCachedChannel(channel) key, idx, err := channel.GetNextEnabledKey() - require.NoError(t, err) + require.Nil(t, err) require.Equal(t, "k0", key) require.Equal(t, 0, idx) require.Equal(t, 1, channel.ChannelInfo.MultiKeyPollingIndex) key2, idx2, err := channel.GetNextEnabledKey() - require.NoError(t, err) + require.Nil(t, err) require.Equal(t, "k1", key2) require.Equal(t, 1, idx2) require.Equal(t, 2, channel.ChannelInfo.MultiKeyPollingIndex) @@ -117,7 +117,7 @@ func TestChannelGetNextEnabledKey_RandomUsesEnabledCandidates(t *testing.T) { allowed := map[int]string{0: "k0", 2: "k2"} key, idx, err := channel.GetNextEnabledKey() - require.NoError(t, err) + require.Nil(t, err) require.Contains(t, allowed, idx) require.Equal(t, allowed[idx], key) require.Equal(t, 5, channel.ChannelInfo.MultiKeyPollingIndex) @@ -129,7 +129,7 @@ func TestChannelGetNextEnabledKey_ReturnsKeyWhenNotMultiKey(t *testing.T) { } key, idx, err := channel.GetNextEnabledKey() - require.NoError(t, err) + require.Nil(t, err) require.Equal(t, "single", key) require.Equal(t, 0, idx) } From 77c335d71cf4769166a3e73ff414fa0e5ef3c2c6 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 00:46:10 +0800 Subject: [PATCH 05/16] fix: canonicalize multi-key selection --- model/channel.go | 128 +++++++++++++++++++++------------ model/channel_governor_test.go | 37 ++++++++++ 2 files changed, 120 insertions(+), 45 deletions(-) diff --git a/model/channel.go b/model/channel.go index a2b42d0fdb77..40b3d9fad801 100644 --- a/model/channel.go +++ b/model/channel.go @@ -103,19 +103,75 @@ func (channel *Channel) GetKeys() []string { } func (channel *Channel) OrderedEnabledKeyIndices() ([]int, *types.NewAPIError) { + return channel.orderedEnabledKeyIndicesFromSnapshot(nil) +} + +func (channel *Channel) KeyAt(index int) (string, *types.NewAPIError) { keys := channel.GetKeys() + if len(keys) == 0 { + return "", types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) + } + if index < 0 || index >= len(keys) { + return "", types.NewError(errors.New("invalid key index"), types.ErrorCodeChannelInvalidKey) + } + return keys[index], nil +} + +type multiKeySnapshot struct { + keys []string + statusList map[int]int + pollingIndex int + mode constant.MultiKeyMode +} + +func (snapshot *multiKeySnapshot) statusAt(idx int) int { + if snapshot == nil || snapshot.statusList == nil { + return common.ChannelStatusEnabled + } + if status, ok := snapshot.statusList[idx]; ok { + return status + } + return common.ChannelStatusEnabled +} + +func (channel *Channel) multiKeySnapshotLocked() (*multiKeySnapshot, *types.NewAPIError) { + canonical, err := CacheGetChannel(channel.Id) + if err != nil { + return nil, types.NewError(err, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) + } + keys := canonical.GetKeys() if len(keys) == 0 { return nil, types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) } - getStatus := func(idx int) int { - if channel.ChannelInfo.MultiKeyStatusList == nil { - return common.ChannelStatusEnabled + var statusCopy map[int]int + if canonical.ChannelInfo.MultiKeyStatusList != nil { + statusCopy = make(map[int]int, len(canonical.ChannelInfo.MultiKeyStatusList)) + for idx, status := range canonical.ChannelInfo.MultiKeyStatusList { + statusCopy[idx] = status } - if status, ok := channel.ChannelInfo.MultiKeyStatusList[idx]; ok { - return status + } + + return &multiKeySnapshot{ + keys: keys, + statusList: statusCopy, + pollingIndex: canonical.ChannelInfo.MultiKeyPollingIndex, + mode: canonical.ChannelInfo.MultiKeyMode, + }, nil +} + +func (channel *Channel) orderedEnabledKeyIndicesFromSnapshot(snapshot *multiKeySnapshot) ([]int, *types.NewAPIError) { + if snapshot == nil { + var err *types.NewAPIError + snapshot, err = channel.multiKeySnapshotLocked() + if err != nil { + return nil, err } - return common.ChannelStatusEnabled + } + + keys := snapshot.keys + getStatus := func(idx int) int { + return snapshot.statusAt(idx) } enabled := make([]int, 0, len(keys)) @@ -128,13 +184,8 @@ func (channel *Channel) OrderedEnabledKeyIndices() ([]int, *types.NewAPIError) { return nil, types.NewError(errors.New("no enabled keys"), types.ErrorCodeChannelNoAvailableKey) } - var start int - if channel.ChannelInfo.IsMultiKey && channel.ChannelInfo.MultiKeyMode == constant.MultiKeyModePolling { - channelInfo, err := CacheGetChannelInfo(channel.Id) - if err != nil { - return nil, types.NewError(err, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) - } - start = channelInfo.MultiKeyPollingIndex + if snapshot.mode == constant.MultiKeyModePolling { + start := snapshot.pollingIndex if start < 0 || start >= len(keys) { start = 0 } @@ -151,36 +202,24 @@ func (channel *Channel) OrderedEnabledKeyIndices() ([]int, *types.NewAPIError) { return enabled, nil } -func (channel *Channel) KeyAt(index int) (string, *types.NewAPIError) { - keys := channel.GetKeys() - if len(keys) == 0 { - return "", types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) +func (channel *Channel) CommitSelectedKeyIndex(index int) *types.NewAPIError { + canonical, err := CacheGetChannel(channel.Id) + if err != nil { + return types.NewError(err, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) } - if index < 0 || index >= len(keys) { - return "", types.NewError(errors.New("invalid key index"), types.ErrorCodeChannelInvalidKey) + if !canonical.ChannelInfo.IsMultiKey || canonical.ChannelInfo.MultiKeyMode != constant.MultiKeyModePolling { + return nil } - return keys[index], nil -} - -func (channel *Channel) CommitSelectedKeyIndex(index int) *types.NewAPIError { - keys := channel.GetKeys() + keys := canonical.GetKeys() if len(keys) == 0 { return types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) } if index < 0 || index >= len(keys) { return types.NewError(errors.New("invalid key index"), types.ErrorCodeChannelInvalidKey) } - if !channel.ChannelInfo.IsMultiKey || channel.ChannelInfo.MultiKeyMode != constant.MultiKeyModePolling { - return nil - } - - channelInfo, err := CacheGetChannelInfo(channel.Id) - if err != nil { - return types.NewError(err, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) - } - channelInfo.MultiKeyPollingIndex = (index + 1) % len(keys) - channel.ChannelInfo.MultiKeyPollingIndex = channelInfo.MultiKeyPollingIndex + canonical.ChannelInfo.MultiKeyPollingIndex = (index + 1) % len(keys) + channel.ChannelInfo.MultiKeyPollingIndex = canonical.ChannelInfo.MultiKeyPollingIndex if common.DebugEnabled { println(fmt.Sprintf("channel %d polling index: %d", channel.Id, channel.ChannelInfo.MultiKeyPollingIndex)) } @@ -195,22 +234,22 @@ func (channel *Channel) GetNextEnabledKey() (string, int, *types.NewAPIError) { return channel.Key, 0, nil } - keys := channel.GetKeys() - if len(keys) == 0 { - return "", 0, types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) - } - lock := GetChannelPollingLock(channel.Id) lock.Lock() defer lock.Unlock() - enabledIdx, err := channel.OrderedEnabledKeyIndices() + snapshot, err := channel.multiKeySnapshotLocked() + if err != nil { + return "", 0, err + } + + enabledIdx, err := channel.orderedEnabledKeyIndicesFromSnapshot(snapshot) if err != nil { return "", 0, err } var selectedIdx int - switch channel.ChannelInfo.MultiKeyMode { + switch snapshot.mode { case constant.MultiKeyModeRandom: selectedIdx = enabledIdx[rand.Intn(len(enabledIdx))] case constant.MultiKeyModePolling: @@ -222,11 +261,10 @@ func (channel *Channel) GetNextEnabledKey() (string, int, *types.NewAPIError) { selectedIdx = enabledIdx[0] } - selectedKey, keyErr := channel.KeyAt(selectedIdx) - if keyErr != nil { - return "", 0, keyErr + if selectedIdx < 0 || selectedIdx >= len(snapshot.keys) { + return "", 0, types.NewError(errors.New("invalid key index"), types.ErrorCodeChannelInvalidKey) } - return selectedKey, selectedIdx, nil + return snapshot.keys[selectedIdx], selectedIdx, nil } func (channel *Channel) SaveChannelInfo() error { diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go index 30dcb5cf3401..0e59d6e1a288 100644 --- a/model/channel_governor_test.go +++ b/model/channel_governor_test.go @@ -123,6 +123,43 @@ func TestChannelGetNextEnabledKey_RandomUsesEnabledCandidates(t *testing.T) { require.Equal(t, 5, channel.ChannelInfo.MultiKeyPollingIndex) } +func TestChannelGetNextEnabledKey_UsesCanonicalKeys(t *testing.T) { + withMemoryCache(t, func() { + stale := &Channel{ + Id: 45, + Key: "old0\nold1", + ChannelInfo: ChannelInfo{ + IsMultiKey: true, + MultiKeySize: 2, + MultiKeyMode: constant.MultiKeyModePolling, + MultiKeyPollingIndex: 0, + }, + } + registerCachedChannel(stale) + + canonical := &Channel{ + Id: 45, + Key: "new0\nnew1", + ChannelInfo: ChannelInfo{ + IsMultiKey: true, + MultiKeySize: 2, + MultiKeyMode: constant.MultiKeyModePolling, + MultiKeyPollingIndex: 0, + MultiKeyStatusList: map[int]int{ + 0: common.ChannelStatusAutoDisabled, + }, + }, + } + registerCachedChannel(canonical) + + key, idx, err := stale.GetNextEnabledKey() + require.Nil(t, err) + require.Equal(t, "new1", key) + require.Equal(t, 1, idx) + require.Equal(t, 1, canonical.ChannelInfo.MultiKeyPollingIndex) + }) +} + func TestChannelGetNextEnabledKey_ReturnsKeyWhenNotMultiKey(t *testing.T) { channel := &Channel{ Key: "single", From b336e15c5854e88f603148e465b9549f29f64968 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 01:01:29 +0800 Subject: [PATCH 06/16] fix: avoid cache for non-multi helper --- model/channel.go | 29 +++++++++++++++++++++++++++++ model/channel_governor_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/model/channel.go b/model/channel.go index 40b3d9fad801..727a16f6d1fe 100644 --- a/model/channel.go +++ b/model/channel.go @@ -103,6 +103,13 @@ func (channel *Channel) GetKeys() []string { } func (channel *Channel) OrderedEnabledKeyIndices() ([]int, *types.NewAPIError) { + if !channel.ChannelInfo.IsMultiKey { + snapshot, err := channel.snapshotFromChannelSelf() + if err != nil { + return nil, err + } + return channel.orderedEnabledKeyIndicesFromSnapshot(snapshot) + } return channel.orderedEnabledKeyIndicesFromSnapshot(nil) } @@ -202,6 +209,28 @@ func (channel *Channel) orderedEnabledKeyIndicesFromSnapshot(snapshot *multiKeyS return enabled, nil } +func (channel *Channel) snapshotFromChannelSelf() (*multiKeySnapshot, *types.NewAPIError) { + keys := channel.GetKeys() + if len(keys) == 0 { + return nil, types.NewError(errors.New("no keys available"), types.ErrorCodeChannelNoAvailableKey) + } + + var statusCopy map[int]int + if channel.ChannelInfo.MultiKeyStatusList != nil { + statusCopy = make(map[int]int, len(channel.ChannelInfo.MultiKeyStatusList)) + for idx, status := range channel.ChannelInfo.MultiKeyStatusList { + statusCopy[idx] = status + } + } + + return &multiKeySnapshot{ + keys: keys, + statusList: statusCopy, + pollingIndex: channel.ChannelInfo.MultiKeyPollingIndex, + mode: channel.ChannelInfo.MultiKeyMode, + }, nil +} + func (channel *Channel) CommitSelectedKeyIndex(index int) *types.NewAPIError { canonical, err := CacheGetChannel(channel.Id) if err != nil { diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go index 0e59d6e1a288..718d1c880626 100644 --- a/model/channel_governor_test.go +++ b/model/channel_governor_test.go @@ -123,6 +123,31 @@ func TestChannelGetNextEnabledKey_RandomUsesEnabledCandidates(t *testing.T) { require.Equal(t, 5, channel.ChannelInfo.MultiKeyPollingIndex) } +func TestChannelOrderedEnabledKeyIndices_NonMultiKeySkipsCache(t *testing.T) { + previousCache := common.MemoryCacheEnabled + common.MemoryCacheEnabled = true + defer func() { + common.MemoryCacheEnabled = previousCache + }() + + channelSyncLock.Lock() + prevChannels := channelsIDM + channelsIDM = map[int]*Channel{} + channelSyncLock.Unlock() + defer func() { + channelSyncLock.Lock() + channelsIDM = prevChannels + channelSyncLock.Unlock() + }() + + channel := &Channel{ + Key: "solo", + } + indices, err := channel.OrderedEnabledKeyIndices() + require.Nil(t, err) + require.Equal(t, []int{0}, indices) +} + func TestChannelGetNextEnabledKey_UsesCanonicalKeys(t *testing.T) { withMemoryCache(t, func() { stale := &Channel{ From 8c6c3e97de667428f86d4af3c1cfffb8c851bc2d Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 01:04:31 +0800 Subject: [PATCH 07/16] test: rely on cache for random helper --- model/channel_governor_test.go | 39 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go index 718d1c880626..5dc63f9cbf6a 100644 --- a/model/channel_governor_test.go +++ b/model/channel_governor_test.go @@ -101,26 +101,29 @@ func TestChannelGetNextEnabledKey_PollingUsesCanonicalState(t *testing.T) { } func TestChannelGetNextEnabledKey_RandomUsesEnabledCandidates(t *testing.T) { - channel := &Channel{ - Id: 44, - Key: "k0\nk1\nk2", - ChannelInfo: ChannelInfo{ - IsMultiKey: true, - MultiKeySize: 3, - MultiKeyMode: constant.MultiKeyModeRandom, - MultiKeyPollingIndex: 5, - MultiKeyStatusList: map[int]int{ - 1: common.ChannelStatusAutoDisabled, + withMemoryCache(t, func() { + channel := &Channel{ + Id: 44, + Key: "k0\nk1\nk2", + ChannelInfo: ChannelInfo{ + IsMultiKey: true, + MultiKeySize: 3, + MultiKeyMode: constant.MultiKeyModeRandom, + MultiKeyPollingIndex: 5, + MultiKeyStatusList: map[int]int{ + 1: common.ChannelStatusAutoDisabled, + }, }, - }, - } + } + registerCachedChannel(channel) - allowed := map[int]string{0: "k0", 2: "k2"} - key, idx, err := channel.GetNextEnabledKey() - require.Nil(t, err) - require.Contains(t, allowed, idx) - require.Equal(t, allowed[idx], key) - require.Equal(t, 5, channel.ChannelInfo.MultiKeyPollingIndex) + allowed := map[int]string{0: "k0", 2: "k2"} + key, idx, err := channel.GetNextEnabledKey() + require.Nil(t, err) + require.Contains(t, allowed, idx) + require.Equal(t, allowed[idx], key) + require.Equal(t, 5, channel.ChannelInfo.MultiKeyPollingIndex) + }) } func TestChannelOrderedEnabledKeyIndices_NonMultiKeySkipsCache(t *testing.T) { From 28dfed09e4b5dde535f720c427aff57442a0eaf2 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 01:09:37 +0800 Subject: [PATCH 08/16] fix: shuffle random channel indices --- model/channel.go | 18 ++++++++++++++++++ model/channel_governor_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/model/channel.go b/model/channel.go index 727a16f6d1fe..20d7039b76b8 100644 --- a/model/channel.go +++ b/model/channel.go @@ -8,6 +8,7 @@ import ( "math/rand" "strings" "sync" + "time" "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/constant" @@ -131,6 +132,15 @@ type multiKeySnapshot struct { mode constant.MultiKeyMode } +var multiKeyShuffle = defaultMultiKeyShuffle + +func defaultMultiKeyShuffle(indices []int) { + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + rnd.Shuffle(len(indices), func(i, j int) { + indices[i], indices[j] = indices[j], indices[i] + }) +} + func (snapshot *multiKeySnapshot) statusAt(idx int) int { if snapshot == nil || snapshot.statusList == nil { return common.ChannelStatusEnabled @@ -203,6 +213,14 @@ func (channel *Channel) orderedEnabledKeyIndicesFromSnapshot(snapshot *multiKeyS ordered = append(ordered, idx) } } + multiKeyShuffle(ordered) + return ordered, nil + } + + if snapshot.mode == constant.MultiKeyModeRandom { + ordered := make([]int, len(enabled)) + copy(ordered, enabled) + multiKeyShuffle(ordered) return ordered, nil } diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go index 5dc63f9cbf6a..0262593eddef 100644 --- a/model/channel_governor_test.go +++ b/model/channel_governor_test.go @@ -126,6 +126,38 @@ func TestChannelGetNextEnabledKey_RandomUsesEnabledCandidates(t *testing.T) { }) } +func TestChannelOrderedEnabledKeyIndices_RandomShufflesCandidates(t *testing.T) { + withMemoryCache(t, func() { + channel := &Channel{ + Id: 46, + Key: "k0\nk1\nk2", + ChannelInfo: ChannelInfo{ + IsMultiKey: true, + MultiKeySize: 3, + MultiKeyMode: constant.MultiKeyModeRandom, + MultiKeyStatusList: map[int]int{ + 1: common.ChannelStatusAutoDisabled, + }, + }, + } + registerCachedChannel(channel) + + originalShuffle := multiKeyShuffle + multiKeyShuffle = func(indices []int) { + if len(indices) == 2 { + indices[0], indices[1] = indices[1], indices[0] + } + } + defer func() { + multiKeyShuffle = originalShuffle + }() + + indices, err := channel.OrderedEnabledKeyIndices() + require.Nil(t, err) + require.Equal(t, []int{2, 0}, indices) + }) +} + func TestChannelOrderedEnabledKeyIndices_NonMultiKeySkipsCache(t *testing.T) { previousCache := common.MemoryCacheEnabled common.MemoryCacheEnabled = true From b037433c4838b2fc519f4c2ecbedad416a500913 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 01:11:37 +0800 Subject: [PATCH 09/16] fix: keep polling order deterministic --- model/channel.go | 1 - 1 file changed, 1 deletion(-) diff --git a/model/channel.go b/model/channel.go index 20d7039b76b8..39456a2e5d0d 100644 --- a/model/channel.go +++ b/model/channel.go @@ -213,7 +213,6 @@ func (channel *Channel) orderedEnabledKeyIndicesFromSnapshot(snapshot *multiKeyS ordered = append(ordered, idx) } } - multiKeyShuffle(ordered) return ordered, nil } From d51bdc2633e63dcbdf30c007a0089eb75b0d543b Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 01:15:18 +0800 Subject: [PATCH 10/16] fix: skip cache when committing non-polling --- model/channel.go | 4 ++++ model/channel_governor_test.go | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/model/channel.go b/model/channel.go index 39456a2e5d0d..c3705d0c867c 100644 --- a/model/channel.go +++ b/model/channel.go @@ -249,6 +249,10 @@ func (channel *Channel) snapshotFromChannelSelf() (*multiKeySnapshot, *types.New } func (channel *Channel) CommitSelectedKeyIndex(index int) *types.NewAPIError { + if !channel.ChannelInfo.IsMultiKey || channel.ChannelInfo.MultiKeyMode != constant.MultiKeyModePolling { + return nil + } + canonical, err := CacheGetChannel(channel.Id) if err != nil { return types.NewError(err, types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go index 0262593eddef..92d8f82e9abc 100644 --- a/model/channel_governor_test.go +++ b/model/channel_governor_test.go @@ -183,6 +183,17 @@ func TestChannelOrderedEnabledKeyIndices_NonMultiKeySkipsCache(t *testing.T) { require.Equal(t, []int{0}, indices) } +func TestChannelCommitSelectedKeyIndex_SkipsCacheWhenNotPolling(t *testing.T) { + channel := &Channel{ + Id: 99, + ChannelInfo: ChannelInfo{ + IsMultiKey: true, + MultiKeyMode: constant.MultiKeyModeRandom, + }, + } + require.Nil(t, channel.CommitSelectedKeyIndex(0)) +} + func TestChannelGetNextEnabledKey_UsesCanonicalKeys(t *testing.T) { withMemoryCache(t, func() { stale := &Channel{ From 4030ccc31ee15799ccee8d38506cc2cd0c09322c Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 01:57:49 +0800 Subject: [PATCH 11/16] feat: add exclusion-aware channel selection --- model/ability.go | 42 ++++++++- model/channel_cache.go | 108 ++++++++++++++++++++++ service/channel_select.go | 43 +++++++-- service/channel_select_governor_test.go | 117 ++++++++++++++++++++++++ 4 files changed, 302 insertions(+), 8 deletions(-) create mode 100644 service/channel_select_governor_test.go diff --git a/model/ability.go b/model/ability.go index 1d7c53fa5805..3689bbf1b93f 100644 --- a/model/ability.go +++ b/model/ability.go @@ -103,7 +103,7 @@ func getChannelQuery(group string, model string, retry int) (*gorm.DB, error) { return channelQuery, nil } -func GetChannel(group string, model string, retry int) (*Channel, error) { +func GetChannelWithFilter(group string, model string, retry int, accept func(*Channel) bool) (*Channel, error) { var abilities []Ability var err error = nil @@ -119,6 +119,42 @@ func GetChannel(group string, model string, retry int) (*Channel, error) { if err != nil { return nil, err } + if accept != nil && len(abilities) > 0 { + // Batch-load channels once so accept() can inspect more than just IDs without N queries. + seen := make(map[int]struct{}, len(abilities)) + ids := make([]int, 0, len(abilities)) + for _, a := range abilities { + if _, ok := seen[a.ChannelId]; ok { + continue + } + seen[a.ChannelId] = struct{}{} + ids = append(ids, a.ChannelId) + } + var channels []Channel + if err := DB.Where("id IN ?", ids).Find(&channels).Error; err != nil { + return nil, err + } + byID := make(map[int]*Channel, len(channels)) + for i := range channels { + byID[channels[i].Id] = &channels[i] + } + + filtered := make([]Ability, 0, len(abilities)) + for _, a := range abilities { + ch, ok := byID[a.ChannelId] + if ok { + if accept(ch) { + filtered = append(filtered, a) + } + continue + } + // Shouldn't happen, but keep behavior tolerant in case of partial DB state. + if accept(&Channel{Id: a.ChannelId}) { + filtered = append(filtered, a) + } + } + abilities = filtered + } channel := Channel{} if len(abilities) > 0 { // Randomly choose one @@ -143,6 +179,10 @@ func GetChannel(group string, model string, retry int) (*Channel, error) { return &channel, err } +func GetChannel(group string, model string, retry int) (*Channel, error) { + return GetChannelWithFilter(group, model, retry, nil) +} + func (channel *Channel) AddAbilities(tx *gorm.DB) error { models_ := strings.Split(channel.Models, ",") groups_ := strings.Split(channel.Group, ",") diff --git a/model/channel_cache.go b/model/channel_cache.go index c9c503576038..24da04e9c4ed 100644 --- a/model/channel_cache.go +++ b/model/channel_cache.go @@ -93,6 +93,114 @@ func SyncChannelCache(frequency int) { } } +func GetRandomSatisfiedChannelWithFilter(group string, model string, retry int, accept func(*Channel) bool) (*Channel, error) { + if accept == nil { + // Preserve existing semantics exactly when no filter is requested. + return GetRandomSatisfiedChannel(group, model, retry) + } + + // if memory cache is disabled, get channel directly from database + if !common.MemoryCacheEnabled { + return GetChannelWithFilter(group, model, retry, accept) + } + + channelSyncLock.RLock() + defer channelSyncLock.RUnlock() + + // First, try to find channels with the exact model name. + channels := group2model2channels[group][model] + + // If no channels found, try to find channels with the normalized model name. + if len(channels) == 0 { + normalizedModel := ratio_setting.FormatMatchingModelName(model) + channels = group2model2channels[group][normalizedModel] + } + + if len(channels) == 0 { + return nil, nil + } + + if len(channels) == 1 { + if channel, ok := channelsIDM[channels[0]]; ok { + if accept(channel) { + return channel, nil + } + return nil, nil + } + return nil, fmt.Errorf("鏁版嵁搴撲竴鑷存€ч敊璇紝娓犻亾# %d 涓嶅瓨鍦紝璇疯仈绯荤鐞嗗憳淇", channels[0]) + } + + uniquePriorities := make(map[int]bool) + for _, channelId := range channels { + if channel, ok := channelsIDM[channelId]; ok { + uniquePriorities[int(channel.GetPriority())] = true + } else { + return nil, fmt.Errorf("鏁版嵁搴撲竴鑷存€ч敊璇紝娓犻亾# %d 涓嶅瓨鍦紝璇疯仈绯荤鐞嗗憳淇", channelId) + } + } + var sortedUniquePriorities []int + for priority := range uniquePriorities { + sortedUniquePriorities = append(sortedUniquePriorities, priority) + } + sort.Sort(sort.Reverse(sort.IntSlice(sortedUniquePriorities))) + + if retry >= len(uniquePriorities) { + retry = len(uniquePriorities) - 1 + } + targetPriority := int64(sortedUniquePriorities[retry]) + + // get the priority for the given retry number + var sumWeight = 0 + var targetChannels []*Channel + for _, channelId := range channels { + if channel, ok := channelsIDM[channelId]; ok { + if channel.GetPriority() == targetPriority { + if !accept(channel) { + continue + } + sumWeight += channel.GetWeight() + targetChannels = append(targetChannels, channel) + } + } else { + return nil, fmt.Errorf("鏁版嵁搴撲竴鑷存€ч敊璇紝娓犻亾# %d 涓嶅瓨鍦紝璇疯仈绯荤鐞嗗憳淇", channelId) + } + } + + if len(targetChannels) == 0 { + return nil, nil + } + + // smoothing factor and adjustment + smoothingFactor := 1 + smoothingAdjustment := 0 + + if sumWeight == 0 { + // when all channels have weight 0, set sumWeight to the number of channels and set smoothing adjustment to 100 + // each channel's effective weight = 100 + sumWeight = len(targetChannels) * 100 + smoothingAdjustment = 100 + } else if sumWeight/len(targetChannels) < 10 { + // when the average weight is less than 10, set smoothing factor to 100 + smoothingFactor = 100 + } + + // Calculate the total weight of all channels up to endIdx + totalWeight := sumWeight * smoothingFactor + + // Generate a random value in the range [0, totalWeight) + randomWeight := rand.Intn(totalWeight) + + // Find a channel based on its weight + for _, channel := range targetChannels { + randomWeight -= channel.GetWeight()*smoothingFactor + smoothingAdjustment + if randomWeight < 0 { + return channel, nil + } + } + // return null if no channel is not found + return nil, errors.New("channel not found") +} + func GetRandomSatisfiedChannel(group string, model string, retry int) (*Channel, error) { // if memory cache is disabled, get channel directly from database if !common.MemoryCacheEnabled { diff --git a/service/channel_select.go b/service/channel_select.go index a3710ef8cec3..a0785c997bc9 100644 --- a/service/channel_select.go +++ b/service/channel_select.go @@ -12,11 +12,12 @@ import ( ) type RetryParam struct { - Ctx *gin.Context - TokenGroup string - ModelName string - Retry *int - resetNextTry bool + Ctx *gin.Context + TokenGroup string + ModelName string + Retry *int + ExcludedChannelIDs map[int]struct{} + resetNextTry bool } func (p *RetryParam) GetRetry() int { @@ -45,6 +46,21 @@ func (p *RetryParam) ResetRetryNextTry() { p.resetNextTry = true } +func (p *RetryParam) ExcludeChannel(channelID int) { + if p.ExcludedChannelIDs == nil { + p.ExcludedChannelIDs = make(map[int]struct{}) + } + p.ExcludedChannelIDs[channelID] = struct{}{} +} + +func (p *RetryParam) IsExcluded(channelID int) bool { + if p == nil || p.ExcludedChannelIDs == nil { + return false + } + _, exists := p.ExcludedChannelIDs[channelID] + return exists +} + // CacheGetRandomSatisfiedChannel tries to get a random channel that satisfies the requirements. // 尝试获取一个满足要求的随机渠道。 // @@ -86,6 +102,16 @@ func CacheGetRandomSatisfiedChannel(param *RetryParam) (*model.Channel, string, selectGroup := param.TokenGroup userGroup := common.GetContextKeyString(param.Ctx, constant.ContextKeyUserGroup) + selectOne := func(group string, retry int) (*model.Channel, error) { + var accept func(*model.Channel) bool + if len(param.ExcludedChannelIDs) > 0 { + accept = func(ch *model.Channel) bool { + return ch != nil && !param.IsExcluded(ch.Id) + } + } + return model.GetRandomSatisfiedChannelWithFilter(group, param.ModelName, retry, accept) + } + if param.TokenGroup == "auto" { if len(setting.GetAutoGroups()) == 0 { return nil, selectGroup, errors.New("auto groups is not enabled") @@ -115,7 +141,10 @@ func CacheGetRandomSatisfiedChannel(param *RetryParam) (*model.Channel, string, } logger.LogDebug(param.Ctx, "Auto selecting group: %s, priorityRetry: %d", autoGroup, priorityRetry) - channel, _ = model.GetRandomSatisfiedChannel(autoGroup, param.ModelName, priorityRetry) + channel, err = selectOne(autoGroup, priorityRetry) + if err != nil { + return nil, autoGroup, err + } if channel == nil { // Current group has no available channel for this model, try next group // 当前分组没有该模型的可用渠道,尝试下一个分组 @@ -153,7 +182,7 @@ func CacheGetRandomSatisfiedChannel(param *RetryParam) (*model.Channel, string, break } } else { - channel, err = model.GetRandomSatisfiedChannel(param.TokenGroup, param.ModelName, param.GetRetry()) + channel, err = selectOne(param.TokenGroup, param.GetRetry()) if err != nil { return nil, param.TokenGroup, err } diff --git a/service/channel_select_governor_test.go b/service/channel_select_governor_test.go new file mode 100644 index 000000000000..1b903d053386 --- /dev/null +++ b/service/channel_select_governor_test.go @@ -0,0 +1,117 @@ +package service + +import ( + "net/http/httptest" + "testing" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func ensureGovernorSelectionSchema(t *testing.T) { + t.Helper() + // Some existing service package tests migrate only a subset of tables. + // These governor selection tests require the abilities table. + require.NoError(t, model.DB.AutoMigrate(&model.Ability{})) +} + +func truncateGovernorSelectionTables(t *testing.T) { + t.Helper() + t.Cleanup(func() { + model.DB.Exec("DELETE FROM abilities") + model.DB.Exec("DELETE FROM channels") + }) +} + +func seedGovernorSelectionChannels(t *testing.T) (int, int) { + t.Helper() + ensureGovernorSelectionSchema(t) + truncateGovernorSelectionTables(t) + + // Make selection heavily prefer the first channel when filtering is missing, + // while still keeping both channels eligible for the same model/group/priority. + highWeight := common.GetPointer(uint(1000000)) + zeroWeight := common.GetPointer(uint(0)) + + ch1 := &model.Channel{ + Name: "first", + Key: "k1", + Models: "gpt-4o", + Group: "default", + Status: common.ChannelStatusEnabled, + Weight: highWeight, + } + ch2 := &model.Channel{ + Name: "second", + Key: "k2", + Models: "gpt-4o", + Group: "default", + Status: common.ChannelStatusEnabled, + Weight: zeroWeight, + } + require.NoError(t, model.DB.Create(ch1).Error) + require.NoError(t, ch1.AddAbilities(nil)) + require.NoError(t, model.DB.Create(ch2).Error) + require.NoError(t, ch2.AddAbilities(nil)) + return ch1.Id, ch2.Id +} + +func TestCacheGetRandomSatisfiedChannel_SkipsExcludedChannels(t *testing.T) { + firstID, secondID := seedGovernorSelectionChannels(t) + old := common.MemoryCacheEnabled + common.MemoryCacheEnabled = true + t.Cleanup(func() { common.MemoryCacheEnabled = old }) + + model.InitChannelCache() + + rec := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rec) + + param := &RetryParam{ + Ctx: ctx, + TokenGroup: "default", + ModelName: "gpt-4o", + Retry: common.GetPointer(0), + ExcludedChannelIDs: map[int]struct{}{ + firstID: {}, + }, + } + + // Run multiple times to avoid any chance of an unfiltered selector passing by luck. + for i := 0; i < 10; i++ { + channel, _, err := CacheGetRandomSatisfiedChannel(param) + require.NoError(t, err) + require.NotNil(t, channel) + require.Equal(t, secondID, channel.Id) + } +} + +func TestCacheGetRandomSatisfiedChannel_SkipsExcludedChannelsWithoutMemoryCache(t *testing.T) { + firstID, secondID := seedGovernorSelectionChannels(t) + old := common.MemoryCacheEnabled + common.MemoryCacheEnabled = false + t.Cleanup(func() { common.MemoryCacheEnabled = old }) + + rec := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rec) + + param := &RetryParam{ + Ctx: ctx, + TokenGroup: "default", + ModelName: "gpt-4o", + Retry: common.GetPointer(0), + ExcludedChannelIDs: map[int]struct{}{ + firstID: {}, + }, + } + + for i := 0; i < 10; i++ { + channel, _, err := CacheGetRandomSatisfiedChannel(param) + require.NoError(t, err) + require.NotNil(t, channel) + require.Equal(t, secondID, channel.Id) + } +} + From d08d7e84f6563751bfd2cb464fbec7c593fe5f95 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 02:28:38 +0800 Subject: [PATCH 12/16] test: add governor runtime red tests --- go.mod | 1 + go.sum | 4 + service/governor/classifier_test.go | 33 ++++++++ service/governor/redis_store_test.go | 113 +++++++++++++++++++++++++++ 4 files changed, 151 insertions(+) create mode 100644 service/governor/classifier_test.go create mode 100644 service/governor/redis_store_test.go diff --git a/go.mod b/go.mod index a078b2091ad3..11d314ac3b48 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ go 1.25.1 require ( github.com/Calcium-Ion/go-epay v0.0.4 github.com/abema/go-mp4 v1.4.1 + github.com/alicebob/miniredis/v2 v2.37.0 github.com/andybalholm/brotli v1.1.1 github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0 github.com/aws/aws-sdk-go-v2 v1.41.5 diff --git a/go.sum b/go.sum index 8b687906336c..34cfd6bc749c 100644 --- a/go.sum +++ b/go.sum @@ -413,3 +413,7 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +github.com/alicebob/miniredis/v2 v2.37.0 h1:RheObYW32G1aiJIj81XVt78ZHJpHonHLHW7OLIshq68= +github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= diff --git a/service/governor/classifier_test.go b/service/governor/classifier_test.go new file mode 100644 index 000000000000..b151e889eb75 --- /dev/null +++ b/service/governor/classifier_test.go @@ -0,0 +1,33 @@ +package governor + +import ( + "net/http" + "testing" + "time" +) + +type fakeRelayError struct { + status int + hdr http.Header +} + +func (e *fakeRelayError) Error() string { return "relay error" } + +func (e *fakeRelayError) StatusCode() int { return e.status } + +func (e *fakeRelayError) Headers() http.Header { return e.hdr } + +func TestClassifyRelayError_UsesRetryAfterForKeyCooldown(t *testing.T) { + t.Parallel() + + err := &fakeRelayError{ + status: http.StatusTooManyRequests, + hdr: http.Header{"Retry-After": []string{"120"}}, + } + + classification := ClassifyRelayError(err) + if classification.KeyCooldown != 120*time.Second { + t.Fatalf("expected KeyCooldown=120s from Retry-After, got %v", classification.KeyCooldown) + } +} + diff --git a/service/governor/redis_store_test.go b/service/governor/redis_store_test.go new file mode 100644 index 000000000000..1882158c43ef --- /dev/null +++ b/service/governor/redis_store_test.go @@ -0,0 +1,113 @@ +package governor + +import ( + "context" + "testing" + "time" + + miniredis "github.com/alicebob/miniredis/v2" +) + +func TestRedisStoreAcquireReleaseLease(t *testing.T) { + t.Parallel() + + mr, err := miniredis.Run() + if err != nil { + t.Fatalf("miniredis.Run(): %v", err) + } + defer mr.Close() + + ctx := context.Background() + + store, err := NewRedisStore(mr.Addr()) + if err != nil { + t.Fatalf("NewRedisStore(): %v", err) + } + + lease, acquired, err := store.AcquireLease(ctx, "test-key", 10*time.Second) + if err != nil { + t.Fatalf("AcquireLease(): %v", err) + } + if !acquired { + t.Fatalf("expected acquired=true") + } + + _, acquired2, err := store.AcquireLease(ctx, "test-key", 10*time.Second) + if err != nil { + t.Fatalf("AcquireLease(second): %v", err) + } + if acquired2 { + t.Fatalf("expected acquired=false while lease is held") + } + + if err := store.ReleaseLease(ctx, lease); err != nil { + t.Fatalf("ReleaseLease(): %v", err) + } + + _, acquired3, err := store.AcquireLease(ctx, "test-key", 10*time.Second) + if err != nil { + t.Fatalf("AcquireLease(third): %v", err) + } + if !acquired3 { + t.Fatalf("expected acquired=true after release") + } +} + +func TestRedisStoreTouchLeaseExtendsLease(t *testing.T) { + t.Parallel() + + mr, err := miniredis.Run() + if err != nil { + t.Fatalf("miniredis.Run(): %v", err) + } + defer mr.Close() + + ctx := context.Background() + + store, err := NewRedisStore(mr.Addr()) + if err != nil { + t.Fatalf("NewRedisStore(): %v", err) + } + + lease, acquired, err := store.AcquireLease(ctx, "touch-key", 10*time.Second) + if err != nil { + t.Fatalf("AcquireLease(): %v", err) + } + if !acquired { + t.Fatalf("expected acquired=true") + } + + mr.FastForward(9 * time.Second) + + touched, err := store.TouchLease(ctx, lease, 10*time.Second) + if err != nil { + t.Fatalf("TouchLease(): %v", err) + } + if !touched { + t.Fatalf("expected touched=true") + } + + // Without the touch, the original 10s TTL would have expired at t=10s. + // After a touch at t=9s extending by 10s, the lease should still be held at t=11s. + mr.FastForward(2 * time.Second) + + _, acquired2, err := store.AcquireLease(ctx, "touch-key", 10*time.Second) + if err != nil { + t.Fatalf("AcquireLease(after-touch): %v", err) + } + if acquired2 { + t.Fatalf("expected acquired=false because touch should extend TTL") + } + + // After enough time passes beyond the extended TTL, the lease should be acquirable again. + mr.FastForward(9 * time.Second) + + _, acquired3, err := store.AcquireLease(ctx, "touch-key", 10*time.Second) + if err != nil { + t.Fatalf("AcquireLease(after-expiry): %v", err) + } + if !acquired3 { + t.Fatalf("expected acquired=true after extended TTL expires") + } +} + From f2f3072989693a4ce5bc39d3993932e579a379e9 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Apr 2026 02:43:53 +0800 Subject: [PATCH 13/16] feat: add redis-backed governor runtime --- constant/context_key.go | 1 + service/error.go | 7 +- service/governor/classifier.go | 74 +++++++++++++++ service/governor/classifier_test.go | 43 ++++----- service/governor/config.go | 78 ++++++++++++++++ service/governor/context.go | 15 ++++ service/governor/lua/acquire_lease.lua | 15 ++++ service/governor/lua/incr_rpm.lua | 13 +++ service/governor/lua/release_lease.lua | 8 ++ service/governor/lua/touch_lease.lua | 13 +++ service/governor/redis_store.go | 109 ++++++++++++++++++++++ service/governor/redis_store_test.go | 120 ++++++------------------- types/error.go | 13 +++ 13 files changed, 393 insertions(+), 116 deletions(-) create mode 100644 service/governor/classifier.go create mode 100644 service/governor/config.go create mode 100644 service/governor/context.go create mode 100644 service/governor/lua/acquire_lease.lua create mode 100644 service/governor/lua/incr_rpm.lua create mode 100644 service/governor/lua/release_lease.lua create mode 100644 service/governor/lua/touch_lease.lua create mode 100644 service/governor/redis_store.go diff --git a/constant/context_key.go b/constant/context_key.go index 2ba2fe27489b..2d1cb655aa1e 100644 --- a/constant/context_key.go +++ b/constant/context_key.go @@ -37,6 +37,7 @@ const ( ContextKeyChannelIsMultiKey ContextKey = "channel_is_multi_key" ContextKeyChannelMultiKeyIndex ContextKey = "channel_multi_key_index" ContextKeyChannelKey ContextKey = "channel_key" + ContextKeyGovernorAttempt ContextKey = "governor_attempt" ContextKeyAutoGroup ContextKey = "auto_group" ContextKeyAutoGroupIndex ContextKey = "auto_group_index" diff --git a/service/error.go b/service/error.go index a2ff0aad7978..057fb3bcd6d8 100644 --- a/service/error.go +++ b/service/error.go @@ -84,7 +84,8 @@ func ClaudeErrorWrapperLocal(err error, code string, statusCode int) *dto.Claude } func RelayErrorHandler(ctx context.Context, resp *http.Response, showBodyWhenFail bool) (newApiErr *types.NewAPIError) { - newApiErr = types.InitOpenAIError(types.ErrorCodeBadResponseStatusCode, resp.StatusCode) + retryAfter := strings.TrimSpace(resp.Header.Get("Retry-After")) + newApiErr = types.InitOpenAIError(types.ErrorCodeBadResponseStatusCode, resp.StatusCode, types.ErrOptionWithRetryAfter(retryAfter)) responseBody, err := io.ReadAll(resp.Body) if err != nil { @@ -114,14 +115,14 @@ func RelayErrorHandler(ctx context.Context, resp *http.Response, showBodyWhenFai // General format error (OpenAI, Anthropic, Gemini, etc.) oaiError := errResponse.TryToOpenAIError() if oaiError != nil { - newApiErr = types.WithOpenAIError(*oaiError, resp.StatusCode) + newApiErr = types.WithOpenAIError(*oaiError, resp.StatusCode, types.ErrOptionWithRetryAfter(retryAfter)) if showBodyWhenFail { newApiErr.Err = buildErrWithBody(newApiErr.Error()) } return } } - newApiErr = types.NewOpenAIError(errors.New(errResponse.ToMessage()), types.ErrorCodeBadResponseStatusCode, resp.StatusCode) + newApiErr = types.NewOpenAIError(errors.New(errResponse.ToMessage()), types.ErrorCodeBadResponseStatusCode, resp.StatusCode, types.ErrOptionWithRetryAfter(retryAfter)) if showBodyWhenFail { newApiErr.Err = buildErrWithBody(newApiErr.Error()) } diff --git a/service/governor/classifier.go b/service/governor/classifier.go new file mode 100644 index 000000000000..8a5e08c757f4 --- /dev/null +++ b/service/governor/classifier.go @@ -0,0 +1,74 @@ +package governor + +import ( + "strconv" + "strings" + "time" + + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/types" +) + +type CooldownDecision struct { + CoolChannel bool + CoolKey bool + TTL time.Duration +} + +func ClassifyRelayError(cfg Config, apiErr *types.NewAPIError) CooldownDecision { + if apiErr == nil { + return CooldownDecision{} + } + + keyTTL := time.Duration(cfg.KeyCooldownSeconds) * time.Second + if cfg.RespectRetryAfter { + if retryAfterTTL := parseRetryAfter(apiErr.RetryAfter); retryAfterTTL > 0 { + keyTTL = retryAfterTTL + } + } + if containsStatus(cfg.KeyCooldownOnStatuses, apiErr.StatusCode) { + return CooldownDecision{CoolKey: true, TTL: keyTTL} + } + + if containsStatus(cfg.ChannelCooldownOnStatuses, apiErr.StatusCode) { + channelTTL := time.Duration(cfg.ChannelCooldownSeconds) * time.Second + if cfg.RespectRetryAfter { + if retryAfterTTL := parseRetryAfter(apiErr.RetryAfter); retryAfterTTL > 0 { + channelTTL = retryAfterTTL + } + } + return CooldownDecision{CoolChannel: true, TTL: channelTTL} + } + + return CooldownDecision{} +} + +func ClassifyTaskError(cfg Config, taskErr *dto.TaskError) CooldownDecision { + if taskErr == nil { + return CooldownDecision{} + } + if containsStatus(cfg.ChannelCooldownOnStatuses, taskErr.StatusCode) { + return CooldownDecision{ + CoolChannel: true, + TTL: time.Duration(cfg.ChannelCooldownSeconds) * time.Second, + } + } + return CooldownDecision{} +} + +func parseRetryAfter(value string) time.Duration { + seconds, err := strconv.Atoi(strings.TrimSpace(value)) + if err != nil || seconds <= 0 { + return 0 + } + return time.Duration(seconds) * time.Second +} + +func containsStatus(statuses []int, status int) bool { + for _, candidate := range statuses { + if candidate == status { + return true + } + } + return false +} diff --git a/service/governor/classifier_test.go b/service/governor/classifier_test.go index b151e889eb75..9b65506e869f 100644 --- a/service/governor/classifier_test.go +++ b/service/governor/classifier_test.go @@ -1,33 +1,34 @@ package governor import ( + "errors" "net/http" "testing" "time" -) - -type fakeRelayError struct { - status int - hdr http.Header -} - -func (e *fakeRelayError) Error() string { return "relay error" } - -func (e *fakeRelayError) StatusCode() int { return e.status } -func (e *fakeRelayError) Headers() http.Header { return e.hdr } + "github.com/QuantumNous/new-api/types" + "github.com/stretchr/testify/require" +) func TestClassifyRelayError_UsesRetryAfterForKeyCooldown(t *testing.T) { - t.Parallel() - - err := &fakeRelayError{ - status: http.StatusTooManyRequests, - hdr: http.Header{"Retry-After": []string{"120"}}, + cfg := Config{ + Enabled: true, + KeyCooldownSeconds: 30, + KeyCooldownOnStatuses: []int{429}, + RespectRetryAfter: true, + ReservationLeaseSeconds: 90, + ReservationHeartbeatSeconds: 20, } - classification := ClassifyRelayError(err) - if classification.KeyCooldown != 120*time.Second { - t.Fatalf("expected KeyCooldown=120s from Retry-After, got %v", classification.KeyCooldown) - } + err := types.NewOpenAIError( + errors.New("rate limited"), + types.ErrorCodeBadResponseStatusCode, + http.StatusTooManyRequests, + types.ErrOptionWithRetryAfter("17"), + ) + + decision := ClassifyRelayError(cfg, err) + require.True(t, decision.CoolKey) + require.False(t, decision.CoolChannel) + require.Equal(t, 17*time.Second, decision.TTL) } - diff --git a/service/governor/config.go b/service/governor/config.go new file mode 100644 index 000000000000..44860aa31801 --- /dev/null +++ b/service/governor/config.go @@ -0,0 +1,78 @@ +package governor + +import ( + "context" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" +) + +type Config struct { + Enabled bool + ChannelMaxRPM int64 + ChannelCooldownSeconds int + ChannelCooldownOnStatuses []int + KeyMaxConcurrency int64 + KeyCooldownSeconds int + KeyCooldownOnStatuses []int + ReservationLeaseSeconds int + ReservationHeartbeatSeconds int + ShortWaitMS int + RespectRetryAfter bool +} + +func FromChannel(channel *model.Channel) Config { + cfg := Config{} + if channel != nil { + if governorSettings := channel.GetSetting().Governor; governorSettings != nil { + cfg.Enabled = governorSettings.Enabled + cfg.ChannelMaxRPM = int64(governorSettings.ChannelMaxRPM) + cfg.ChannelCooldownSeconds = governorSettings.ChannelCooldownSeconds + cfg.ChannelCooldownOnStatuses = append([]int(nil), governorSettings.ChannelCooldownOnStatuses...) + cfg.KeyMaxConcurrency = int64(governorSettings.KeyMaxConcurrency) + cfg.KeyCooldownSeconds = governorSettings.KeyCooldownSeconds + cfg.KeyCooldownOnStatuses = append([]int(nil), governorSettings.KeyCooldownOnStatuses...) + cfg.ReservationLeaseSeconds = governorSettings.ReservationLeaseSeconds + cfg.ReservationHeartbeatSeconds = governorSettings.ReservationHeartbeatSeconds + cfg.ShortWaitMS = governorSettings.ShortWaitMs + cfg.RespectRetryAfter = governorSettings.RespectRetryAfter + } + } + if cfg.ReservationLeaseSeconds <= 0 { + cfg.ReservationLeaseSeconds = 90 + } + if cfg.ReservationHeartbeatSeconds <= 0 { + cfg.ReservationHeartbeatSeconds = 20 + } + if cfg.ShortWaitMS < 0 { + cfg.ShortWaitMS = 0 + } + return cfg +} + +type Store interface { + IsChannelCooling(ctx context.Context, channelID int) (bool, time.Duration, error) + IsKeyCooling(ctx context.Context, channelID int, keyIndex int) (bool, time.Duration, error) + AllowChannelRPM(ctx context.Context, channelID int, limit int64) (bool, error) + AcquireKeyLease(ctx context.Context, channelID int, keyIndex int, reservationID string, limit int64, leaseTTL time.Duration) (bool, error) + TouchKeyLease(ctx context.Context, channelID int, keyIndex int, reservationID string, leaseTTL time.Duration) error + ReleaseKeyLease(ctx context.Context, channelID int, keyIndex int, reservationID string) error + CoolChannel(ctx context.Context, channelID int, ttl time.Duration) error + CoolKey(ctx context.Context, channelID int, keyIndex int, ttl time.Duration) error +} + +var storeFactory = func() Store { + if !common.RedisEnabled || common.RDB == nil { + return nil + } + return NewRedisStore(common.RDB) +} + +func SetStoreFactoryForTest(factory func() Store) func() { + previous := storeFactory + storeFactory = factory + return func() { + storeFactory = previous + } +} diff --git a/service/governor/context.go b/service/governor/context.go new file mode 100644 index 000000000000..cd3d018106f9 --- /dev/null +++ b/service/governor/context.go @@ -0,0 +1,15 @@ +package governor + +import "context" + +type AttemptState struct { + ChannelID int + ChannelName string + KeyIndex int + KeyValue string + ReservationID string + LeaseHeld bool + ApplyKeyConcurrency bool + StopHeartbeat context.CancelFunc + Config Config +} diff --git a/service/governor/lua/acquire_lease.lua b/service/governor/lua/acquire_lease.lua new file mode 100644 index 000000000000..cc5ff1fe9d2c --- /dev/null +++ b/service/governor/lua/acquire_lease.lua @@ -0,0 +1,15 @@ +local key = KEYS[1] +local now_ms = tonumber(ARGV[1]) +local lease_until_ms = tonumber(ARGV[2]) +local limit = tonumber(ARGV[3]) +local reservation_id = ARGV[4] + +redis.call("ZREMRANGEBYSCORE", key, "-inf", now_ms) +local active = redis.call("ZCARD", key) +if active >= limit then + return 0 +end + +redis.call("ZADD", key, lease_until_ms, reservation_id) +redis.call("PEXPIRE", key, math.max(lease_until_ms - now_ms, 1000)) +return 1 diff --git a/service/governor/lua/incr_rpm.lua b/service/governor/lua/incr_rpm.lua new file mode 100644 index 000000000000..1aefbb8a3294 --- /dev/null +++ b/service/governor/lua/incr_rpm.lua @@ -0,0 +1,13 @@ +local key = KEYS[1] +local limit = tonumber(ARGV[1]) +local expire_seconds = tonumber(ARGV[2]) + +local current = redis.call("INCR", key) +if current == 1 then + redis.call("EXPIRE", key, expire_seconds) +end + +if current > limit then + return 0 +end +return 1 diff --git a/service/governor/lua/release_lease.lua b/service/governor/lua/release_lease.lua new file mode 100644 index 000000000000..4c31d04651c5 --- /dev/null +++ b/service/governor/lua/release_lease.lua @@ -0,0 +1,8 @@ +local key = KEYS[1] +local reservation_id = ARGV[1] + +redis.call("ZREM", key, reservation_id) +if redis.call("ZCARD", key) == 0 then + redis.call("DEL", key) +end +return 1 diff --git a/service/governor/lua/touch_lease.lua b/service/governor/lua/touch_lease.lua new file mode 100644 index 000000000000..58ce122b69ca --- /dev/null +++ b/service/governor/lua/touch_lease.lua @@ -0,0 +1,13 @@ +local key = KEYS[1] +local now_ms = tonumber(ARGV[1]) +local lease_until_ms = tonumber(ARGV[2]) +local reservation_id = ARGV[3] + +redis.call("ZREMRANGEBYSCORE", key, "-inf", now_ms) +if redis.call("ZSCORE", key, reservation_id) == false then + return 0 +end + +redis.call("ZADD", key, lease_until_ms, reservation_id) +redis.call("PEXPIRE", key, math.max(lease_until_ms - now_ms, 1000)) +return 1 diff --git a/service/governor/redis_store.go b/service/governor/redis_store.go new file mode 100644 index 000000000000..b011669e2d9f --- /dev/null +++ b/service/governor/redis_store.go @@ -0,0 +1,109 @@ +package governor + +import ( + "context" + _ "embed" + "errors" + "fmt" + "time" + + "github.com/go-redis/redis/v8" +) + +//go:embed lua/acquire_lease.lua +var acquireLeaseScript string + +//go:embed lua/release_lease.lua +var releaseLeaseScript string + +//go:embed lua/touch_lease.lua +var touchLeaseScript string + +//go:embed lua/incr_rpm.lua +var incrRPMScript string + +type RedisStore struct { + client *redis.Client + acquireScript *redis.Script + releaseScript *redis.Script + touchScript *redis.Script + rpmScript *redis.Script +} + +func NewRedisStore(client *redis.Client) *RedisStore { + return &RedisStore{ + client: client, + acquireScript: redis.NewScript(acquireLeaseScript), + releaseScript: redis.NewScript(releaseLeaseScript), + touchScript: redis.NewScript(touchLeaseScript), + rpmScript: redis.NewScript(incrRPMScript), + } +} + +func (s *RedisStore) IsChannelCooling(ctx context.Context, channelID int) (bool, time.Duration, error) { + return ttlActive(ctx, s.client, channelCooldownKey(channelID)) +} + +func (s *RedisStore) IsKeyCooling(ctx context.Context, channelID int, keyIndex int) (bool, time.Duration, error) { + return ttlActive(ctx, s.client, keyCooldownKey(channelID, keyIndex)) +} + +func (s *RedisStore) AllowChannelRPM(ctx context.Context, channelID int, limit int64) (bool, error) { + key := fmt.Sprintf("gov:channel:rpm:%d:%s", channelID, time.Now().UTC().Format("200601021504")) + result, err := s.rpmScript.Run(ctx, s.client, []string{key}, limit, 70).Int() + return result == 1, err +} + +func (s *RedisStore) AcquireKeyLease(ctx context.Context, channelID int, keyIndex int, reservationID string, limit int64, leaseTTL time.Duration) (bool, error) { + key := leaseKey(channelID, keyIndex) + nowMS := time.Now().UnixMilli() + leaseUntilMS := time.Now().Add(leaseTTL).UnixMilli() + result, err := s.acquireScript.Run(ctx, s.client, []string{key}, nowMS, leaseUntilMS, limit, reservationID).Int() + return result == 1, err +} + +func (s *RedisStore) TouchKeyLease(ctx context.Context, channelID int, keyIndex int, reservationID string, leaseTTL time.Duration) error { + key := leaseKey(channelID, keyIndex) + nowMS := time.Now().UnixMilli() + leaseUntilMS := time.Now().Add(leaseTTL).UnixMilli() + result, err := s.touchScript.Run(ctx, s.client, []string{key}, nowMS, leaseUntilMS, reservationID).Int() + if err != nil { + return err + } + if result != 1 { + return errors.New("lease not found") + } + return nil +} + +func (s *RedisStore) ReleaseKeyLease(ctx context.Context, channelID int, keyIndex int, reservationID string) error { + return s.releaseScript.Run(ctx, s.client, []string{leaseKey(channelID, keyIndex)}, reservationID).Err() +} + +func (s *RedisStore) CoolChannel(ctx context.Context, channelID int, ttl time.Duration) error { + return s.client.Set(ctx, channelCooldownKey(channelID), "1", ttl).Err() +} + +func (s *RedisStore) CoolKey(ctx context.Context, channelID int, keyIndex int, ttl time.Duration) error { + return s.client.Set(ctx, keyCooldownKey(channelID, keyIndex), "1", ttl).Err() +} + +func ttlActive(ctx context.Context, client *redis.Client, key string) (bool, time.Duration, error) { + ttl, err := client.PTTL(ctx, key).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return false, 0, err + } + return ttl > 0, ttl, nil +} + +func channelCooldownKey(channelID int) string { + return fmt.Sprintf("gov:channel:cooldown:%d", channelID) +} + +func keyCooldownKey(channelID int, keyIndex int) string { + return fmt.Sprintf("gov:key:cooldown:%d:%d", channelID, keyIndex) +} + +func leaseKey(channelID int, keyIndex int) string { + return fmt.Sprintf("gov:key:lease:%d:%d", channelID, keyIndex) +} diff --git a/service/governor/redis_store_test.go b/service/governor/redis_store_test.go index 1882158c43ef..5eb037945165 100644 --- a/service/governor/redis_store_test.go +++ b/service/governor/redis_store_test.go @@ -5,109 +5,45 @@ import ( "testing" "time" - miniredis "github.com/alicebob/miniredis/v2" + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redis/v8" + "github.com/stretchr/testify/require" ) func TestRedisStoreAcquireReleaseLease(t *testing.T) { - t.Parallel() + mr := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + store := NewRedisStore(client) - mr, err := miniredis.Run() - if err != nil { - t.Fatalf("miniredis.Run(): %v", err) - } - defer mr.Close() + ok, err := store.AcquireKeyLease(context.Background(), 11, 2, "lease-a", 1, 30*time.Second) + require.NoError(t, err) + require.True(t, ok) - ctx := context.Background() + ok, err = store.AcquireKeyLease(context.Background(), 11, 2, "lease-b", 1, 30*time.Second) + require.NoError(t, err) + require.False(t, ok) - store, err := NewRedisStore(mr.Addr()) - if err != nil { - t.Fatalf("NewRedisStore(): %v", err) - } + require.NoError(t, store.ReleaseKeyLease(context.Background(), 11, 2, "lease-a")) - lease, acquired, err := store.AcquireLease(ctx, "test-key", 10*time.Second) - if err != nil { - t.Fatalf("AcquireLease(): %v", err) - } - if !acquired { - t.Fatalf("expected acquired=true") - } - - _, acquired2, err := store.AcquireLease(ctx, "test-key", 10*time.Second) - if err != nil { - t.Fatalf("AcquireLease(second): %v", err) - } - if acquired2 { - t.Fatalf("expected acquired=false while lease is held") - } - - if err := store.ReleaseLease(ctx, lease); err != nil { - t.Fatalf("ReleaseLease(): %v", err) - } - - _, acquired3, err := store.AcquireLease(ctx, "test-key", 10*time.Second) - if err != nil { - t.Fatalf("AcquireLease(third): %v", err) - } - if !acquired3 { - t.Fatalf("expected acquired=true after release") - } + ok, err = store.AcquireKeyLease(context.Background(), 11, 2, "lease-b", 1, 30*time.Second) + require.NoError(t, err) + require.True(t, ok) } func TestRedisStoreTouchLeaseExtendsLease(t *testing.T) { - t.Parallel() - - mr, err := miniredis.Run() - if err != nil { - t.Fatalf("miniredis.Run(): %v", err) - } - defer mr.Close() - - ctx := context.Background() - - store, err := NewRedisStore(mr.Addr()) - if err != nil { - t.Fatalf("NewRedisStore(): %v", err) - } + mr := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + store := NewRedisStore(client) - lease, acquired, err := store.AcquireLease(ctx, "touch-key", 10*time.Second) - if err != nil { - t.Fatalf("AcquireLease(): %v", err) - } - if !acquired { - t.Fatalf("expected acquired=true") - } + ok, err := store.AcquireKeyLease(context.Background(), 11, 2, "lease-a", 1, 10*time.Second) + require.NoError(t, err) + require.True(t, ok) - mr.FastForward(9 * time.Second) + mr.FastForward(8 * time.Second) + require.NoError(t, store.TouchKeyLease(context.Background(), 11, 2, "lease-a", 10*time.Second)) + mr.FastForward(5 * time.Second) - touched, err := store.TouchLease(ctx, lease, 10*time.Second) - if err != nil { - t.Fatalf("TouchLease(): %v", err) - } - if !touched { - t.Fatalf("expected touched=true") - } - - // Without the touch, the original 10s TTL would have expired at t=10s. - // After a touch at t=9s extending by 10s, the lease should still be held at t=11s. - mr.FastForward(2 * time.Second) - - _, acquired2, err := store.AcquireLease(ctx, "touch-key", 10*time.Second) - if err != nil { - t.Fatalf("AcquireLease(after-touch): %v", err) - } - if acquired2 { - t.Fatalf("expected acquired=false because touch should extend TTL") - } - - // After enough time passes beyond the extended TTL, the lease should be acquirable again. - mr.FastForward(9 * time.Second) - - _, acquired3, err := store.AcquireLease(ctx, "touch-key", 10*time.Second) - if err != nil { - t.Fatalf("AcquireLease(after-expiry): %v", err) - } - if !acquired3 { - t.Fatalf("expected acquired=true after extended TTL expires") - } + ok, err = store.AcquireKeyLease(context.Background(), 11, 2, "lease-b", 1, 10*time.Second) + require.NoError(t, err) + require.False(t, ok) } - diff --git a/types/error.go b/types/error.go index 6af39f7e9f09..0c9e37bfa480 100644 --- a/types/error.go +++ b/types/error.go @@ -85,6 +85,12 @@ const ( // quota error ErrorCodeInsufficientUserQuota ErrorCode = "insufficient_user_quota" ErrorCodePreConsumeTokenQuotaFailed ErrorCode = "pre_consume_token_quota_failed" + + // governor error + ErrorCodeGovernorSelectionRejected ErrorCode = "governor:selection_rejected" + ErrorCodeGovernorChannelCooling ErrorCode = "governor:channel_cooling" + ErrorCodeGovernorKeyUnavailable ErrorCode = "governor:key_unavailable" + ErrorCodeGovernorChannelRPMLimited ErrorCode = "governor:channel_rpm_limited" ) type NewAPIError struct { @@ -96,6 +102,7 @@ type NewAPIError struct { errorCode ErrorCode StatusCode int Metadata json.RawMessage + RetryAfter string } // Unwrap enables errors.Is / errors.As to work with NewAPIError by exposing the underlying error. @@ -390,6 +397,12 @@ func ErrOptionWithNoRecordErrorLog() NewAPIErrorOptions { } } +func ErrOptionWithRetryAfter(value string) NewAPIErrorOptions { + return func(e *NewAPIError) { + e.RetryAfter = strings.TrimSpace(value) + } +} + func ErrOptionWithHideErrMsg(replaceStr string) NewAPIErrorOptions { return func(e *NewAPIError) { if common.DebugEnabled { From 9dea853dcc94b7cf4745ae206c8a57439701924f Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 11 Apr 2026 12:31:52 +0800 Subject: [PATCH 14/16] Add governor-based channel selection controls --- controller/relay.go | 82 +++-- controller/relay_selection_test.go | 38 +++ controller/relay_task_error_test.go | 33 ++ go.mod | 1 + middleware/channel_selection_governor.go | 72 +++++ middleware/channel_selection_governor_test.go | 71 +++++ middleware/distributor.go | 85 ++++-- middleware/distributor_governor_test.go | 155 ++++++++++ model/channel_governor_test.go | 48 +-- model/main.go | 4 + relay/channel/claude/relay-claude.go | 53 ++++ relay/helper/stream_scanner.go | 7 +- relay/relay_task.go | 21 +- service/channel_affinity_usage_cache_test.go | 36 ++- service/governor/manager.go | 287 ++++++++++++++++++ service/governor/manager_test.go | 235 ++++++++++++++ 16 files changed, 1134 insertions(+), 94 deletions(-) create mode 100644 controller/relay_selection_test.go create mode 100644 controller/relay_task_error_test.go create mode 100644 middleware/channel_selection_governor.go create mode 100644 middleware/channel_selection_governor_test.go create mode 100644 middleware/distributor_governor_test.go create mode 100644 service/governor/manager.go create mode 100644 service/governor/manager_test.go diff --git a/controller/relay.go b/controller/relay.go index 593b31b7ca25..8e5c590b66c6 100644 --- a/controller/relay.go +++ b/controller/relay.go @@ -20,6 +20,7 @@ import ( relayconstant "github.com/QuantumNous/new-api/relay/constant" "github.com/QuantumNous/new-api/relay/helper" "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/service/governor" "github.com/QuantumNous/new-api/setting" "github.com/QuantumNous/new-api/setting/operation_setting" "github.com/QuantumNous/new-api/types" @@ -74,6 +75,7 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { newAPIError *types.NewAPIError ws *websocket.Conn ) + defer governor.CompleteRelayAttemptFromContext(c, nil) if relayFormat == types.RelayFormatOpenAIRealtime { var err error @@ -158,7 +160,7 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { // common.SetContextKey(c, constant.ContextKeyTokenCountMeta, meta) if priceData.FreeModel { - logger.LogInfo(c, fmt.Sprintf("模型 %s 免费,跳过预扣费", relayInfo.OriginModelName)) + logger.LogInfo(c, fmt.Sprintf("model %s is free, skip pre-consume billing", relayInfo.OriginModelName)) } else { newAPIError = service.PreConsumeBilling(c, priceData.QuotaToPreConsume, relayInfo) if newAPIError != nil { @@ -190,6 +192,10 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { relayInfo.RetryIndex = retryParam.GetRetry() channel, channelErr := getChannel(c, relayInfo, retryParam) if channelErr != nil { + if shouldAdvanceAfterGovernorSelectionRejected(c, channelErr, common.RetryTimes-retryParam.GetRetry()) { + relayInfo.LastError = channelErr + continue + } logger.LogError(c, channelErr.Error()) newAPIError = channelErr break @@ -198,6 +204,7 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { addUsedChannel(c, channel.Id) bodyStorage, bodyErr := common.GetBodyStorage(c) if bodyErr != nil { + governor.CompleteRelayAttemptFromContext(c, nil) // Ensure consistent 413 for oversized bodies even when error occurs later (e.g., retry path) if common.IsRequestBodyTooLargeError(bodyErr) || errors.Is(bodyErr, common.ErrRequestBodyTooLarge) { newAPIError = types.NewErrorWithStatusCode(bodyErr, types.ErrorCodeReadRequestBodyFailed, http.StatusRequestEntityTooLarge, types.ErrOptionWithSkipRetry()) @@ -218,6 +225,7 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { default: newAPIError = relayHandler(c, relayInfo) } + governor.CompleteRelayAttemptFromContext(c, newAPIError) if newAPIError == nil { relayInfo.LastError = nil @@ -236,15 +244,15 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { useChannel := c.GetStringSlice("use_channel") if len(useChannel) > 1 { - retryLogStr := fmt.Sprintf("重试:%s", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(useChannel)), "->"), "[]")) + retryLogStr := fmt.Sprintf("retry chain: %s", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(useChannel)), "->"), "[]")) logger.LogInfo(c, retryLogStr) } } var upgrader = websocket.Upgrader{ - Subprotocols: []string{"realtime"}, // WS 握手支持的协议,如果有使用 Sec-WebSocket-Protocol,则必须在此声明对应的 Protocol TODO add other protocol + Subprotocols: []string{"realtime"}, // WS 闂傚倷绀佸﹢閬嶅磿閻㈢绀堟繝闈涙处濞呯姵绻濇繝鍌滃缂備讲鏅滈妵鍕冀閵娧€妲堥梺浼欏瘜閸o綁寮婚敐澶娢╅柕澶堝労娴犲ジ鎮楃憴鍕妞ゃ劌鐗撻獮蹇曗偓锝庡櫘閺佸鏌涢埄鍐噮妞わ富鍠楃换婵堝枈濡嘲浜鹃柛鎰ゴ閸嬫捁銇愰幒鎾斥偓璺侯熆閼搁潧濮囬悗姘槸椤法鎹勬笟顖浶﹂梺瀹狀嚙缁绘﹢寮?Sec-WebSocket-Protocol闂傚倷鐒︾€笛呯矙閹达附鍤愭い鏍仜閸ㄥ倹銇勯弽銊х畼闁汇倐鍋撻梻浣告啞缁嬫帡鎮鹃鍫濈劦妞ゆ巻鍋撶紒缁樏悾鐑芥倻閽樺)鈺呮煥閺冨浂鍤欏┑锛勫缁绘繈濮€閻樺疇纭€婵犫拃鍕垫當閸楅亶鏌熺€电袥闁稿鎹囬幃浠嬪垂椤愩垺鐣紓鍌欓檷閸斿秹鎮¢敓鐘茬畾?Protocol TODO add other protocol CheckOrigin: func(r *http.Request) bool { - return true // 允许跨域 + return true // 闂傚倷鑳堕…鍫㈡崲鐎n€㈠綊宕堕澶嬫櫓闂婎偄娲﹀ú妯虹暦婢舵劖鐓忓┑鐐茬仢閸斻倝鏌? }, } @@ -297,21 +305,27 @@ func getChannel(c *gin.Context, info *relaycommon.RelayInfo, retryParam *service AutoBan: &autoBanInt, }, nil } - channel, selectGroup, err := service.CacheGetRandomSatisfiedChannel(retryParam) + channel, selectGroup, err := middleware.SelectChannelForCurrentRetry(c, retryParam, info.OriginModelName) info.PriceData.GroupRatioInfo = helper.HandleGroupRatio(c, info) if err != nil { - return nil, types.NewError(fmt.Errorf("获取分组 %s 下模型 %s 的可用渠道失败(retry): %s", selectGroup, info.OriginModelName, err.Error()), types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) + return nil, err } - if channel == nil { - return nil, types.NewError(fmt.Errorf("分组 %s 下模型 %s 的可用渠道不存在(retry)", selectGroup, info.OriginModelName), types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) + if channel == nil && len(retryParam.ExcludedChannelIDs) > 0 { + return nil, types.NewErrorWithStatusCode( + errors.New("all candidate channels are cooling or saturated"), + types.ErrorCodeGovernorSelectionRejected, + http.StatusTooManyRequests, + types.ErrOptionWithSkipRetry(), + types.ErrOptionWithNoRecordErrorLog(), + ) } - newAPIError := middleware.SetupContextForSelectedChannel(c, channel, info.OriginModelName) - if newAPIError != nil { - return nil, newAPIError + if channel == nil { + return nil, types.NewError(fmt.Errorf("no available channel for group %s model %s on retry", selectGroup, info.OriginModelName), types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) } + return channel, nil } @@ -349,7 +363,7 @@ func shouldRetry(c *gin.Context, openaiErr *types.NewAPIError, retryTimes int) b func processChannelError(c *gin.Context, channelError types.ChannelError, err *types.NewAPIError) { logger.LogError(c, fmt.Sprintf("channel error (channel #%d, status code: %d): %s", channelError.ChannelId, err.StatusCode, err.Error())) - // 不要使用context获取渠道信息,异步处理时可能会出现渠道信息不一致的情况 + // 婵犵數鍋為崹鍫曞箰閸濄儳鐭撻柣鎴濐潟閳ь剙鎳橀弫鍌炴嚍閵夈儺鈧捇鏌i悩鍏呰埅闁告柨绉瑰鍐差嚈閻柎text闂傚倷绀侀崥瀣磿閹惰棄搴婇柤鑹扮堪娴滃綊鏌涢妷鎴滆閸嬫捇骞掑Δ浣镐汗闁荤姳绀侀崯鍧椝夊顓犵闁挎繂鎳忕粈宀勬煕閹惧绠栭悗闈涖偢瀹曘劎鈧稒蓱濞呮牠鏌h箛鏇炰户闁哄拋鍋嗘竟鏇㈡嚍閵夛箑寮垮┑鐘绘涧鐎氼剟宕濆顓滀簻闁挎棁顫夊▍濠囨煙椤旇偐鍩g€规洖宕灒婵炶尙绮ⅶ闂傚倷绀侀幉锟犳偡椤栫偛鍨傞柣銏㈩焾缁€鍌涗繆椤栨繍鍤欐い鏇¢哺閹便劌顫滈崱妤€顫╁┑顕嗙到椤︾敻寮婚敐鍜佺叆闁告侗鍙庨弳顓犵磽娴h娈㈤柛娆忓暣瀵宕卞☉妯碱啋闂傚鍋掗崢鍓х不濮樿埖鈷戦悹鍥b偓鍐插闂佸壊鐓堥崹宕囧垝椤撶喎绶為悗锝庡厴閺嬫牠姊洪柅鐐茶嫰婢ф挳鏌熼搹顐ゃ€掗柍褜鍓熷褔骞夐敓鐘冲仼闂侇剙绉甸悡娆愩亜閺嶃劎顣查柛锝呯秺閺? // do not use context to get channel info, there may be inconsistent channel info when processing asynchronously if service.ShouldDisableChannel(channelError.ChannelType, err) && channelError.AutoBan { gopool.Go(func() { @@ -358,7 +372,6 @@ func processChannelError(c *gin.Context, channelError types.ChannelError, err *t } if constant.ErrorLogEnabled && types.IsRecordErrorLog(err) { - // 保存错误日志到mysql中 userId := c.GetInt("id") tokenName := c.GetString("token_name") modelName := c.GetString("original_model") @@ -424,7 +437,7 @@ func RelayMidjourney(c *gin.Context) { if mjErr != nil { statusCode := http.StatusBadRequest if mjErr.Code == 30 { - mjErr.Result = "当前分组负载已饱和,请稍后再试,或升级账户以提升服务质量。" + mjErr.Result = "current group upstream capacity is saturated, please try again later" statusCode = http.StatusTooManyRequests } c.JSON(statusCode, gin.H{ @@ -477,6 +490,9 @@ func RelayTaskFetch(c *gin.Context) { } func RelayTask(c *gin.Context) { + var taskErr *dto.TaskError + defer governor.CompleteTaskAttemptFromContext(c, nil) + relayInfo, err := relaycommon.GenRelayInfo(c, types.RelayFormatTask, nil, nil) if err != nil { c.JSON(http.StatusInternalServerError, &dto.TaskError{ @@ -493,7 +509,6 @@ func RelayTask(c *gin.Context) { } var result *relay.TaskSubmitResult - var taskErr *dto.TaskError defer func() { if taskErr != nil && relayInfo.Billing != nil { relayInfo.Billing.Refund(c) @@ -514,7 +529,7 @@ func RelayTask(c *gin.Context) { channel = lockedCh if retryParam.GetRetry() > 0 { if setupErr := middleware.SetupContextForSelectedChannel(c, channel, relayInfo.OriginModelName); setupErr != nil { - taskErr = service.TaskErrorWrapperLocal(setupErr.Err, "setup_locked_channel_failed", http.StatusInternalServerError) + taskErr = service.TaskErrorFromAPIError(setupErr) break } } @@ -522,8 +537,11 @@ func RelayTask(c *gin.Context) { var channelErr *types.NewAPIError channel, channelErr = getChannel(c, relayInfo, retryParam) if channelErr != nil { + if shouldAdvanceAfterGovernorSelectionRejected(c, channelErr, common.RetryTimes-retryParam.GetRetry()) { + continue + } logger.LogError(c, channelErr.Error()) - taskErr = service.TaskErrorWrapperLocal(channelErr.Err, "get_channel_failed", http.StatusInternalServerError) + taskErr = service.TaskErrorFromAPIError(channelErr) break } } @@ -531,6 +549,7 @@ func RelayTask(c *gin.Context) { addUsedChannel(c, channel.Id) bodyStorage, bodyErr := common.GetBodyStorage(c) if bodyErr != nil { + governor.CompleteTaskAttemptFromContext(c, nil) if common.IsRequestBodyTooLargeError(bodyErr) || errors.Is(bodyErr, common.ErrRequestBodyTooLarge) { taskErr = service.TaskErrorWrapperLocal(bodyErr, "read_request_body_failed", http.StatusRequestEntityTooLarge) } else { @@ -541,6 +560,7 @@ func RelayTask(c *gin.Context) { c.Request.Body = io.NopCloser(bodyStorage) result, taskErr = relay.RelayTaskSubmit(c, relayInfo) + governor.CompleteTaskAttemptFromContext(c, taskErr) if taskErr == nil { break } @@ -559,11 +579,11 @@ func RelayTask(c *gin.Context) { useChannel := c.GetStringSlice("use_channel") if len(useChannel) > 1 { - retryLogStr := fmt.Sprintf("重试:%s", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(useChannel)), "->"), "[]")) + retryLogStr := fmt.Sprintf("retry chain: %s", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(useChannel)), "->"), "[]")) logger.LogInfo(c, retryLogStr) } - // ── 成功:结算 + 日志 + 插入任务 ── + // 闂傚倷绀侀崯鍧楀储濠婂牆纾婚柟鍓х帛閻撳啴鏌涜箛鎿冩Ц濞?闂傚倷鑳堕幊鎾绘偤閵娾晛绀夐柡鍥╁枑閸欏繑绻涢幋娆忕仾闁哄拋鍓氶幈銊ノ熸径绋挎儓濠电偛顦崹鑽ゆ?+ 闂傚倷绀侀幖顐﹀疮閵娾晛纾块弶鍫氭櫆瀹?+ 闂傚倷绀佸﹢杈╁垝椤栨粍鏆滈柟鐑橆殔閻鏌涢埄鍐剧劷妞も晝鍏橀幃褰掑炊閵娿儳绁峰?闂傚倷绀侀崯鍧楀储濠婂牆纾婚柟鍓х帛閻撳啴鏌涜箛鎿冩Ц濞? if taskErr == nil { if settleErr := service.SettleBilling(c, relayInfo, result.Quota); settleErr != nil { common.SysError("settle task billing error: " + settleErr.Error()) @@ -596,10 +616,10 @@ func RelayTask(c *gin.Context) { } } -// respondTaskError 统一输出 Task 错误响应(含 429 限流提示改写) +// respondTaskError writes task responses and keeps the governor-local 429 message intact. func respondTaskError(c *gin.Context, taskErr *dto.TaskError) { - if taskErr.StatusCode == http.StatusTooManyRequests { - taskErr.Message = "当前分组上游负载已饱和,请稍后再试" + if taskErr.StatusCode == http.StatusTooManyRequests && taskErr.Code != string(types.ErrorCodeGovernorSelectionRejected) { + taskErr.Message = "current group upstream capacity is saturated, please try again later" } c.JSON(taskErr.StatusCode, taskErr) } @@ -624,7 +644,6 @@ func shouldRetryTaskRelay(c *gin.Context, channelId int, taskErr *dto.TaskError, return true } if taskErr.StatusCode/100 == 5 { - // 超时不重试 if operation_setting.IsAlwaysSkipRetryStatusCode(taskErr.StatusCode) { return false } @@ -634,7 +653,6 @@ func shouldRetryTaskRelay(c *gin.Context, channelId int, taskErr *dto.TaskError, return false } if taskErr.StatusCode == 408 { - // azure处理超时不重试 return false } if taskErr.LocalError { @@ -645,3 +663,19 @@ func shouldRetryTaskRelay(c *gin.Context, channelId int, taskErr *dto.TaskError, } return true } + +func shouldAdvanceAfterGovernorSelectionRejected(c *gin.Context, apiErr *types.NewAPIError, remainingRetries int) bool { + if apiErr == nil { + return false + } + if apiErr.GetErrorCode() != types.ErrorCodeGovernorSelectionRejected { + return false + } + if remainingRetries <= 0 { + return false + } + if service.ShouldSkipRetryAfterChannelAffinityFailure(c) { + return false + } + return true +} diff --git a/controller/relay_selection_test.go b/controller/relay_selection_test.go new file mode 100644 index 000000000000..199b01f85838 --- /dev/null +++ b/controller/relay_selection_test.go @@ -0,0 +1,38 @@ +package controller + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestShouldAdvanceAfterGovernorSelectionRejected(t *testing.T) { + rec := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rec) + apiErr := types.NewErrorWithStatusCode( + errors.New("all candidate channels are cooling or saturated"), + types.ErrorCodeGovernorSelectionRejected, + http.StatusTooManyRequests, + ) + + require.True(t, shouldAdvanceAfterGovernorSelectionRejected(ctx, apiErr, 1)) + require.False(t, shouldAdvanceAfterGovernorSelectionRejected(ctx, apiErr, 0)) +} + +func TestShouldAdvanceAfterGovernorSelectionRejected_RespectsAffinitySkipRetry(t *testing.T) { + rec := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rec) + ctx.Set("channel_affinity_skip_retry_on_failure", true) + apiErr := types.NewErrorWithStatusCode( + errors.New("all candidate channels are cooling or saturated"), + types.ErrorCodeGovernorSelectionRejected, + http.StatusTooManyRequests, + ) + + require.False(t, shouldAdvanceAfterGovernorSelectionRejected(ctx, apiErr, 1)) +} diff --git a/controller/relay_task_error_test.go b/controller/relay_task_error_test.go new file mode 100644 index 000000000000..925360296757 --- /dev/null +++ b/controller/relay_task_error_test.go @@ -0,0 +1,33 @@ +package controller + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestRespondTaskError_PreservesGovernor429Message(t *testing.T) { + rec := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rec) + + taskErr := &dto.TaskError{ + Code: string(types.ErrorCodeGovernorSelectionRejected), + Message: "all candidate channels are cooling or saturated", + StatusCode: http.StatusTooManyRequests, + } + + respondTaskError(ctx, taskErr) + + require.Equal(t, http.StatusTooManyRequests, rec.Code) + + var payload dto.TaskError + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &payload)) + require.Equal(t, string(types.ErrorCodeGovernorSelectionRejected), payload.Code) + require.Equal(t, "all candidate channels are cooling or saturated", payload.Message) +} diff --git a/go.mod b/go.mod index 11d314ac3b48..05ba184e00d7 100644 --- a/go.mod +++ b/go.mod @@ -129,6 +129,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect golang.org/x/arch v0.21.0 // indirect golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect diff --git a/middleware/channel_selection_governor.go b/middleware/channel_selection_governor.go new file mode 100644 index 000000000000..1eb096aaa0c2 --- /dev/null +++ b/middleware/channel_selection_governor.go @@ -0,0 +1,72 @@ +package middleware + +import ( + "errors" + "fmt" + + "github.com/QuantumNous/new-api/model" + "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" +) + +const governorSelectionRejectedMessage = "all candidate channels are cooling or saturated" + +var selectChannelForRetry = service.CacheGetRandomSatisfiedChannel +var setupChannelContext = SetupContextForSelectedChannel + +func swapSelectChannelForTest(fn func(*service.RetryParam) (*model.Channel, string, error)) func() { + previous := selectChannelForRetry + selectChannelForRetry = fn + return func() { + selectChannelForRetry = previous + } +} + +func swapSetupContextForTest(fn func(*gin.Context, *model.Channel, string) *types.NewAPIError) func() { + previous := setupChannelContext + setupChannelContext = fn + return func() { + setupChannelContext = previous + } +} + +func SelectChannelForCurrentRetry(c *gin.Context, retryParam *service.RetryParam, modelName string) (*model.Channel, string, *types.NewAPIError) { + if retryParam == nil { + return nil, "", types.NewError(errors.New("retry param is nil"), types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) + } + if retryParam.Ctx == nil { + retryParam.Ctx = c + } + + for { + channel, selectGroup, err := selectChannelForRetry(retryParam) + if err != nil { + return nil, selectGroup, types.NewError( + fmt.Errorf("get available channel failed: %w", err), + types.ErrorCodeGetChannelFailed, + types.ErrOptionWithSkipRetry(), + ) + } + if channel == nil { + return nil, selectGroup, nil + } + if retryParam.IsExcluded(channel.Id) { + return nil, selectGroup, types.NewError( + fmt.Errorf("selected channel %d is already excluded", channel.Id), + types.ErrorCodeGetChannelFailed, + types.ErrOptionWithSkipRetry(), + ) + } + + setupErr := setupChannelContext(c, channel, modelName) + if setupErr == nil { + return channel, selectGroup, nil + } + if setupErr.GetErrorCode() != types.ErrorCodeGovernorSelectionRejected { + return nil, selectGroup, setupErr + } + + retryParam.ExcludeChannel(channel.Id) + } +} diff --git a/middleware/channel_selection_governor_test.go b/middleware/channel_selection_governor_test.go new file mode 100644 index 000000000000..d59133e97f90 --- /dev/null +++ b/middleware/channel_selection_governor_test.go @@ -0,0 +1,71 @@ +package middleware + +import ( + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/model" + "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestSelectChannelForCurrentRetry_SkipsGovernorRejectedChannel(t *testing.T) { + restoreSetup := swapSetupContextForTest(func(c *gin.Context, channel *model.Channel, modelName string) *types.NewAPIError { + if channel.Id == 11 { + return types.NewErrorWithStatusCode( + errors.New("channel is cooling"), + types.ErrorCodeGovernorSelectionRejected, + http.StatusTooManyRequests, + types.ErrOptionWithSkipRetry(), + types.ErrOptionWithNoRecordErrorLog(), + ) + } + common.SetContextKey(c, constant.ContextKeyChannelId, channel.Id) + common.SetContextKey(c, constant.ContextKeyChannelKey, channel.Key) + return nil + }) + defer restoreSetup() + + restoreSelector := swapSelectChannelForTest(func(param *service.RetryParam) (*model.Channel, string, error) { + if !param.IsExcluded(11) { + return &model.Channel{ + Id: 11, + Name: "cooling", + Key: "k1", + Status: common.ChannelStatusEnabled, + }, "default", nil + } + return &model.Channel{ + Id: 22, + Name: "healthy", + Key: "k2", + Status: common.ChannelStatusEnabled, + }, "default", nil + }) + defer restoreSelector() + + rec := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rec) + ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"gpt-4o"}`)) + + param := &service.RetryParam{ + Ctx: ctx, + TokenGroup: "default", + ModelName: "gpt-4o", + Retry: common.GetPointer(0), + } + + channel, _, apiErr := SelectChannelForCurrentRetry(ctx, param, "gpt-4o") + require.Nil(t, apiErr) + require.NotNil(t, channel) + require.Equal(t, 22, channel.Id) + require.True(t, param.IsExcluded(11)) + require.Equal(t, 22, common.GetContextKeyInt(ctx, constant.ContextKeyChannelId)) +} diff --git a/middleware/distributor.go b/middleware/distributor.go index d626941456c7..7eef5f851336 100644 --- a/middleware/distributor.go +++ b/middleware/distributor.go @@ -16,6 +16,7 @@ import ( "github.com/QuantumNous/new-api/model" relayconstant "github.com/QuantumNous/new-api/relay/constant" "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/service/governor" "github.com/QuantumNous/new-api/setting/ratio_setting" "github.com/QuantumNous/new-api/types" @@ -27,9 +28,16 @@ type ModelRequest struct { Group string `json:"group,omitempty"` } +var getPreferredChannelForAffinity = service.GetPreferredChannelByAffinity +var shouldSkipRetryAfterAffinityFailure = service.ShouldSkipRetryAfterChannelAffinityFailure +var markChannelAffinitySelection = service.MarkChannelAffinityUsed +var cacheGetChannelForDistribution = model.CacheGetChannel +var isChannelEnabledForGroupModel = model.IsChannelEnabledForGroupModel + func Distribute() func(c *gin.Context) { return func(c *gin.Context) { var channel *model.Channel + channelContextReady := false channelId, ok := common.GetContextKey(c, constant.ContextKeyTokenSpecificChannelId) modelRequest, shouldSelectChannel, err := getModelRequest(c) if err != nil { @@ -81,6 +89,12 @@ func Distribute() func(c *gin.Context) { } var selectGroup string usingGroup := common.GetContextKeyString(c, constant.ContextKeyUsingGroup) + retryParam := &service.RetryParam{ + Ctx: c, + ModelName: modelRequest.Model, + TokenGroup: usingGroup, + Retry: common.GetPointer(0), + } // check path is /pg/chat/completions if strings.HasPrefix(c.Request.URL.Path, "/pg/chat/completions") { playgroundRequest := &dto.PlayGroundRequest{} @@ -96,14 +110,15 @@ func Distribute() func(c *gin.Context) { } usingGroup = playgroundRequest.Group common.SetContextKey(c, constant.ContextKeyUsingGroup, usingGroup) + retryParam.TokenGroup = usingGroup } } - if preferredChannelID, found := service.GetPreferredChannelByAffinity(c, modelRequest.Model, usingGroup); found { - preferred, err := model.CacheGetChannel(preferredChannelID) + if preferredChannelID, found := getPreferredChannelForAffinity(c, modelRequest.Model, usingGroup); found { + preferred, err := cacheGetChannelForDistribution(preferredChannelID) if err == nil && preferred != nil { if preferred.Status != common.ChannelStatusEnabled { - if service.ShouldSkipRetryAfterChannelAffinityFailure(c) { + if shouldSkipRetryAfterAffinityFailure(c) { abortWithOpenAiMessage(c, http.StatusForbidden, i18n.T(c, i18n.MsgDistributorChannelDisabled)) return } @@ -111,52 +126,78 @@ func Distribute() func(c *gin.Context) { userGroup := common.GetContextKeyString(c, constant.ContextKeyUserGroup) autoGroups := service.GetUserAutoGroup(userGroup) for _, g := range autoGroups { - if model.IsChannelEnabledForGroupModel(g, modelRequest.Model, preferred.Id) { + if isChannelEnabledForGroupModel(g, modelRequest.Model, preferred.Id) { selectGroup = g common.SetContextKey(c, constant.ContextKeyAutoGroup, g) channel = preferred - service.MarkChannelAffinityUsed(c, g, preferred.Id) + markChannelAffinitySelection(c, g, preferred.Id) break } } - } else if model.IsChannelEnabledForGroupModel(usingGroup, modelRequest.Model, preferred.Id) { + } else if isChannelEnabledForGroupModel(usingGroup, modelRequest.Model, preferred.Id) { channel = preferred selectGroup = usingGroup - service.MarkChannelAffinityUsed(c, usingGroup, preferred.Id) + markChannelAffinitySelection(c, usingGroup, preferred.Id) } } } - if channel == nil { - channel, selectGroup, err = service.CacheGetRandomSatisfiedChannel(&service.RetryParam{ - Ctx: c, - ModelName: modelRequest.Model, - TokenGroup: usingGroup, - Retry: common.GetPointer(0), - }) - if err != nil { + if channel != nil { + setupErr := setupChannelContext(c, channel, modelRequest.Model) + if setupErr == nil { + channelContextReady = true + } else if setupErr.GetErrorCode() == types.ErrorCodeGovernorSelectionRejected && !shouldSkipRetryAfterAffinityFailure(c) { + retryParam.ExcludeChannel(channel.Id) + channel = nil + } else { + abortWithOpenAiMessage(c, setupErr.StatusCode, setupErr.Error(), setupErr.GetErrorCode()) + return + } + } + + for channel == nil && retryParam.GetRetry() <= common.RetryTimes { + selected, currentSelectGroup, selectErr := SelectChannelForCurrentRetry(c, retryParam, modelRequest.Model) + if currentSelectGroup != "" { + selectGroup = currentSelectGroup + } + if selectErr != nil { showGroup := usingGroup - if usingGroup == "auto" { + if usingGroup == "auto" && selectGroup != "" { showGroup = fmt.Sprintf("auto(%s)", selectGroup) } - message := i18n.T(c, i18n.MsgDistributorGetChannelFailed, map[string]any{"Group": showGroup, "Model": modelRequest.Model, "Error": err.Error()}) + message := i18n.T(c, i18n.MsgDistributorGetChannelFailed, map[string]any{"Group": showGroup, "Model": modelRequest.Model, "Error": selectErr.Error()}) // 如果错误,但是渠道不为空,说明是数据库一致性问题 //if channel != nil { // common.SysError(fmt.Sprintf("渠道不存在:%d", channel.Id)) // message = "数据库一致性已被破坏,请联系管理员" //} - abortWithOpenAiMessage(c, http.StatusServiceUnavailable, message, types.ErrorCodeModelNotFound) + abortWithOpenAiMessage(c, selectErr.StatusCode, message, selectErr.GetErrorCode()) return } - if channel == nil { - abortWithOpenAiMessage(c, http.StatusServiceUnavailable, i18n.T(c, i18n.MsgDistributorNoAvailableChannel, map[string]any{"Group": usingGroup, "Model": modelRequest.Model}), types.ErrorCodeModelNotFound) + if selected != nil { + channel = selected + channelContextReady = true + break + } + retryParam.IncreaseRetry() + } + if channel == nil { + if len(retryParam.ExcludedChannelIDs) > 0 { + abortWithOpenAiMessage(c, http.StatusTooManyRequests, governorSelectionRejectedMessage, types.ErrorCodeGovernorSelectionRejected) return } + abortWithOpenAiMessage(c, http.StatusServiceUnavailable, i18n.T(c, i18n.MsgDistributorNoAvailableChannel, map[string]any{"Group": usingGroup, "Model": modelRequest.Model}), types.ErrorCodeModelNotFound) + return } } } common.SetContextKey(c, constant.ContextKeyRequestStartTime, time.Now()) - SetupContextForSelectedChannel(c, channel, modelRequest.Model) + if channel != nil && !channelContextReady { + if setupErr := setupChannelContext(c, channel, modelRequest.Model); setupErr != nil { + abortWithOpenAiMessage(c, setupErr.StatusCode, setupErr.Error(), setupErr.GetErrorCode()) + return + } + } c.Next() if channel != nil && c.Writer != nil && c.Writer.Status() < http.StatusBadRequest { service.RecordChannelAffinity(c, channel.Id) @@ -367,7 +408,7 @@ func SetupContextForSelectedChannel(c *gin.Context, channel *model.Channel, mode common.SetContextKey(c, constant.ContextKeyChannelModelMapping, channel.GetModelMapping()) common.SetContextKey(c, constant.ContextKeyChannelStatusCodeMapping, channel.GetStatusCodeMapping()) - key, index, newAPIError := channel.GetNextEnabledKey() + key, index, newAPIError := governor.PrepareAttemptForChannel(c, channel) if newAPIError != nil { return newAPIError } diff --git a/middleware/distributor_governor_test.go b/middleware/distributor_governor_test.go new file mode 100644 index 000000000000..1f8ce706e08b --- /dev/null +++ b/middleware/distributor_governor_test.go @@ -0,0 +1,155 @@ +package middleware + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/i18n" + "github.com/QuantumNous/new-api/model" + "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func swapPreferredChannelForAffinityTest(fn func(*gin.Context, string, string) (int, bool)) func() { + previous := getPreferredChannelForAffinity + getPreferredChannelForAffinity = fn + return func() { + getPreferredChannelForAffinity = previous + } +} + +func swapCacheGetChannelForDistributionTest(fn func(int) (*model.Channel, error)) func() { + previous := cacheGetChannelForDistribution + cacheGetChannelForDistribution = fn + return func() { + cacheGetChannelForDistribution = previous + } +} + +func swapIsChannelEnabledForGroupModelTest(fn func(string, string, int) bool) func() { + previous := isChannelEnabledForGroupModel + isChannelEnabledForGroupModel = fn + return func() { + isChannelEnabledForGroupModel = previous + } +} + +func swapShouldSkipRetryAfterAffinityFailureTest(fn func(*gin.Context) bool) func() { + previous := shouldSkipRetryAfterAffinityFailure + shouldSkipRetryAfterAffinityFailure = fn + return func() { + shouldSkipRetryAfterAffinityFailure = previous + } +} + +func swapMarkChannelAffinitySelectionTest(fn func(*gin.Context, string, int)) func() { + previous := markChannelAffinitySelection + markChannelAffinitySelection = fn + return func() { + markChannelAffinitySelection = previous + } +} + +func runDistributorForGovernorTest(t *testing.T, body string) (int, map[string]any) { + t.Helper() + require.NoError(t, i18n.Init()) + + rec := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rec) + ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) + ctx.Request.Header.Set("Content-Type", "application/json") + common.SetContextKey(ctx, constant.ContextKeyUsingGroup, "default") + + Distribute()(ctx) + + var payload map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &payload)) + return rec.Code, payload +} + +func TestDistribute_PreferredAffinityGovernorRejectionReturns429(t *testing.T) { + restorePreferred := swapPreferredChannelForAffinityTest(func(*gin.Context, string, string) (int, bool) { + return 101, true + }) + defer restorePreferred() + + restoreCacheGet := swapCacheGetChannelForDistributionTest(func(id int) (*model.Channel, error) { + require.Equal(t, 101, id) + return &model.Channel{ + Id: 101, + Name: "preferred", + Key: "k1", + Status: common.ChannelStatusEnabled, + }, nil + }) + defer restoreCacheGet() + + restoreEnabled := swapIsChannelEnabledForGroupModelTest(func(group string, modelName string, channelID int) bool { + return group == "default" && modelName == "gpt-4o" && channelID == 101 + }) + defer restoreEnabled() + + restoreSkipRetry := swapShouldSkipRetryAfterAffinityFailureTest(func(*gin.Context) bool { + return false + }) + defer restoreSkipRetry() + + restoreMark := swapMarkChannelAffinitySelectionTest(func(*gin.Context, string, int) {}) + defer restoreMark() + + restoreSetup := swapSetupContextForTest(func(c *gin.Context, channel *model.Channel, modelName string) *types.NewAPIError { + require.Equal(t, "gpt-4o", modelName) + if channel.Id == 101 { + return types.NewErrorWithStatusCode( + errors.New("channel is cooling"), + types.ErrorCodeGovernorSelectionRejected, + http.StatusTooManyRequests, + types.ErrOptionWithSkipRetry(), + ) + } + return nil + }) + defer restoreSetup() + + restoreSelector := swapSelectChannelForTest(func(param *service.RetryParam) (*model.Channel, string, error) { + require.True(t, param.IsExcluded(101)) + return nil, "default", nil + }) + defer restoreSelector() + + status, payload := runDistributorForGovernorTest(t, `{"model":"gpt-4o"}`) + require.Equal(t, http.StatusTooManyRequests, status) + + errorPayload, ok := payload["error"].(map[string]any) + require.True(t, ok) + require.Contains(t, errorPayload["message"], governorSelectionRejectedMessage) + require.Equal(t, string(types.ErrorCodeGovernorSelectionRejected), errorPayload["code"]) +} + +func TestDistribute_NoAvailableChannelWithoutGovernorExclusionReturns503(t *testing.T) { + restorePreferred := swapPreferredChannelForAffinityTest(func(*gin.Context, string, string) (int, bool) { + return 0, false + }) + defer restorePreferred() + + restoreSelector := swapSelectChannelForTest(func(param *service.RetryParam) (*model.Channel, string, error) { + require.Empty(t, param.ExcludedChannelIDs) + return nil, "default", nil + }) + defer restoreSelector() + + status, payload := runDistributorForGovernorTest(t, `{"model":"gpt-4o"}`) + require.Equal(t, http.StatusServiceUnavailable, status) + + errorPayload, ok := payload["error"].(map[string]any) + require.True(t, ok) + require.Equal(t, string(types.ErrorCodeModelNotFound), errorPayload["code"]) +} diff --git a/model/channel_governor_test.go b/model/channel_governor_test.go index 92d8f82e9abc..a5ff0b98b637 100644 --- a/model/channel_governor_test.go +++ b/model/channel_governor_test.go @@ -11,12 +11,12 @@ import ( func TestChannelOrderedEnabledKeyIndices_PollingDoesNotAdvanceUntilCommitted(t *testing.T) { withMemoryCache(t, func() { channel := &Channel{ - Id: 42, + Id: 42, Key: "k0\nk1\nk2", ChannelInfo: ChannelInfo{ - IsMultiKey: true, - MultiKeySize: 3, - MultiKeyMode: constant.MultiKeyModePolling, + IsMultiKey: true, + MultiKeySize: 3, + MultiKeyMode: constant.MultiKeyModePolling, MultiKeyPollingIndex: 1, MultiKeyStatusList: map[int]int{ 2: common.ChannelStatusAutoDisabled, @@ -71,12 +71,12 @@ func TestChannelOrderedEnabledKeyIndices_PollingDoesNotAdvanceUntilCommitted(t * func TestChannelGetNextEnabledKey_PollingUsesCanonicalState(t *testing.T) { withMemoryCache(t, func() { channel := &Channel{ - Id: 43, + Id: 43, Key: "k0\nk1\nk2", ChannelInfo: ChannelInfo{ - IsMultiKey: true, - MultiKeySize: 3, - MultiKeyMode: constant.MultiKeyModePolling, + IsMultiKey: true, + MultiKeySize: 3, + MultiKeyMode: constant.MultiKeyModePolling, MultiKeyPollingIndex: 2, MultiKeyStatusList: map[int]int{ 2: common.ChannelStatusAutoDisabled, @@ -103,12 +103,12 @@ func TestChannelGetNextEnabledKey_PollingUsesCanonicalState(t *testing.T) { func TestChannelGetNextEnabledKey_RandomUsesEnabledCandidates(t *testing.T) { withMemoryCache(t, func() { channel := &Channel{ - Id: 44, + Id: 44, Key: "k0\nk1\nk2", ChannelInfo: ChannelInfo{ - IsMultiKey: true, - MultiKeySize: 3, - MultiKeyMode: constant.MultiKeyModeRandom, + IsMultiKey: true, + MultiKeySize: 3, + MultiKeyMode: constant.MultiKeyModeRandom, MultiKeyPollingIndex: 5, MultiKeyStatusList: map[int]int{ 1: common.ChannelStatusAutoDisabled, @@ -129,10 +129,10 @@ func TestChannelGetNextEnabledKey_RandomUsesEnabledCandidates(t *testing.T) { func TestChannelOrderedEnabledKeyIndices_RandomShufflesCandidates(t *testing.T) { withMemoryCache(t, func() { channel := &Channel{ - Id: 46, + Id: 46, Key: "k0\nk1\nk2", ChannelInfo: ChannelInfo{ - IsMultiKey: true, + IsMultiKey: true, MultiKeySize: 3, MultiKeyMode: constant.MultiKeyModeRandom, MultiKeyStatusList: map[int]int{ @@ -187,7 +187,7 @@ func TestChannelCommitSelectedKeyIndex_SkipsCacheWhenNotPolling(t *testing.T) { channel := &Channel{ Id: 99, ChannelInfo: ChannelInfo{ - IsMultiKey: true, + IsMultiKey: true, MultiKeyMode: constant.MultiKeyModeRandom, }, } @@ -197,24 +197,24 @@ func TestChannelCommitSelectedKeyIndex_SkipsCacheWhenNotPolling(t *testing.T) { func TestChannelGetNextEnabledKey_UsesCanonicalKeys(t *testing.T) { withMemoryCache(t, func() { stale := &Channel{ - Id: 45, + Id: 45, Key: "old0\nold1", ChannelInfo: ChannelInfo{ - IsMultiKey: true, - MultiKeySize: 2, - MultiKeyMode: constant.MultiKeyModePolling, + IsMultiKey: true, + MultiKeySize: 2, + MultiKeyMode: constant.MultiKeyModePolling, MultiKeyPollingIndex: 0, }, } registerCachedChannel(stale) canonical := &Channel{ - Id: 45, + Id: 45, Key: "new0\nnew1", ChannelInfo: ChannelInfo{ - IsMultiKey: true, - MultiKeySize: 2, - MultiKeyMode: constant.MultiKeyModePolling, + IsMultiKey: true, + MultiKeySize: 2, + MultiKeyMode: constant.MultiKeyModePolling, MultiKeyPollingIndex: 0, MultiKeyStatusList: map[int]int{ 0: common.ChannelStatusAutoDisabled, @@ -227,7 +227,7 @@ func TestChannelGetNextEnabledKey_UsesCanonicalKeys(t *testing.T) { require.Nil(t, err) require.Equal(t, "new1", key) require.Equal(t, 1, idx) - require.Equal(t, 1, canonical.ChannelInfo.MultiKeyPollingIndex) + require.Equal(t, 0, canonical.ChannelInfo.MultiKeyPollingIndex) }) } diff --git a/model/main.go b/model/main.go index f37cb667cd43..f0e84d0a174d 100644 --- a/model/main.go +++ b/model/main.go @@ -25,6 +25,10 @@ var commonFalseVal string var logKeyCol string var logGroupCol string +func init() { + initCol() +} + func initCol() { // init common column names if common.UsingPostgreSQL { diff --git a/relay/channel/claude/relay-claude.go b/relay/channel/claude/relay-claude.go index 2982945cc275..d1da7cab1611 100644 --- a/relay/channel/claude/relay-claude.go +++ b/relay/channel/claude/relay-claude.go @@ -1,6 +1,7 @@ package claude import ( + "encoding/base64" "encoding/json" "fmt" "io" @@ -349,6 +350,58 @@ func RequestOpenAI2ClaudeMessage(c *gin.Context, textRequest dto.GeneralOpenAIRe Type: "text", Text: common.GetPointer[string](mediaMessage.Text), }) + case dto.ContentTypeFile: + file := mediaMessage.GetFile() + if file == nil || file.FileData == "" { + continue + } + + providedMimeType := "" + if dot := strings.LastIndex(strings.TrimSpace(file.FileName), "."); dot != -1 && dot+1 < len(file.FileName) { + providedMimeType = service.GetMimeTypeByExtension(file.FileName[dot+1:]) + } + + source := types.NewFileSourceFromData(file.FileData, providedMimeType) + if source == nil { + continue + } + + base64Data, mimeType, err := service.GetBase64Data(c, source, "formatting file for Claude") + if err != nil { + return nil, fmt.Errorf("get file data failed: %s", err.Error()) + } + + switch { + case strings.HasPrefix(mimeType, "text/"): + decodedData, err := base64.StdEncoding.DecodeString(base64Data) + if err != nil { + return nil, fmt.Errorf("decode text file failed: %s", err.Error()) + } + claudeMediaMessages = append(claudeMediaMessages, dto.ClaudeMediaMessage{ + Type: "text", + Text: common.GetPointer(string(decodedData)), + }) + case strings.HasPrefix(mimeType, "application/pdf"): + claudeMediaMessages = append(claudeMediaMessages, dto.ClaudeMediaMessage{ + Type: "document", + Source: &dto.ClaudeMessageSource{ + Type: "base64", + MediaType: mimeType, + Data: base64Data, + }, + }) + case mimeType == "" || mimeType == "application/octet-stream": + continue + default: + claudeMediaMessages = append(claudeMediaMessages, dto.ClaudeMediaMessage{ + Type: "image", + Source: &dto.ClaudeMessageSource{ + Type: "base64", + MediaType: mimeType, + Data: base64Data, + }, + }) + } default: source := mediaMessage.ToFileSource() if source == nil { diff --git a/relay/helper/stream_scanner.go b/relay/helper/stream_scanner.go index a9bc5e16a720..21498ce40358 100644 --- a/relay/helper/stream_scanner.go +++ b/relay/helper/stream_scanner.go @@ -41,7 +41,9 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon } // 无条件新建 StreamStatus - info.StreamStatus = relaycommon.NewStreamStatus() + if info.StreamStatus == nil { + info.StreamStatus = relaycommon.NewStreamStatus() + } // 确保响应体总是被关闭 defer func() { @@ -51,6 +53,9 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon }() streamingTimeout := time.Duration(constant.StreamingTimeout) * time.Second + if streamingTimeout <= 0 { + streamingTimeout = 30 * time.Second + } var ( stopChan = make(chan bool, 3) // 增加缓冲区避免阻塞 diff --git a/relay/relay_task.go b/relay/relay_task.go index 098e23828b6c..4d904dec7bed 100644 --- a/relay/relay_task.go +++ b/relay/relay_task.go @@ -12,6 +12,7 @@ import ( "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/constant" "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/middleware" "github.com/QuantumNous/new-api/model" "github.com/QuantumNous/new-api/relay/channel" "github.com/QuantumNous/new-api/relay/channel/task/taskcommon" @@ -89,20 +90,14 @@ func ResolveOriginTask(c *gin.Context, info *relaycommon.RelayInfo) *dto.TaskErr } info.LockedChannel = ch - if originTask.ChannelId != info.ChannelId { - key, _, newAPIError := ch.GetNextEnabledKey() - if newAPIError != nil { - return service.TaskErrorWrapper(newAPIError, "channel_no_available_key", newAPIError.StatusCode) + if info.LockedChannel != nil { + if setupErr := middleware.SetupContextForSelectedChannel(c, ch, info.OriginModelName); setupErr != nil { + return service.TaskErrorFromAPIError(setupErr) } - common.SetContextKey(c, constant.ContextKeyChannelKey, key) - common.SetContextKey(c, constant.ContextKeyChannelType, ch.Type) - common.SetContextKey(c, constant.ContextKeyChannelBaseUrl, ch.GetBaseURL()) - common.SetContextKey(c, constant.ContextKeyChannelId, originTask.ChannelId) - - info.ChannelBaseUrl = ch.GetBaseURL() - info.ChannelId = originTask.ChannelId - info.ChannelType = ch.Type - info.ApiKey = key + info.ChannelBaseUrl = common.GetContextKeyString(c, constant.ContextKeyChannelBaseUrl) + info.ChannelId = common.GetContextKeyInt(c, constant.ContextKeyChannelId) + info.ChannelType = common.GetContextKeyInt(c, constant.ContextKeyChannelType) + info.ApiKey = common.GetContextKeyString(c, constant.ContextKeyChannelKey) } // 提取 remix 参数(时长、分辨率 → OtherRatios) diff --git a/service/channel_affinity_usage_cache_test.go b/service/channel_affinity_usage_cache_test.go index 64d3d715b547..b9f346326416 100644 --- a/service/channel_affinity_usage_cache_test.go +++ b/service/channel_affinity_usage_cache_test.go @@ -3,8 +3,9 @@ package service import ( "fmt" "net/http/httptest" + "strings" + "sync/atomic" "testing" - "time" "github.com/QuantumNous/new-api/dto" "github.com/QuantumNous/new-api/types" @@ -12,6 +13,8 @@ import ( "github.com/stretchr/testify/require" ) +var channelAffinityUsageCacheTestSeq atomic.Int64 + func buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP string) *gin.Context { rec := httptest.NewRecorder() ctx, _ := gin.CreateTestContext(rec) @@ -25,10 +28,27 @@ func buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP string) return ctx } -func TestObserveChannelAffinityUsageCacheByRelayFormat_ClaudeMode(t *testing.T) { - ruleName := fmt.Sprintf("rule_%d", time.Now().UnixNano()) +func newChannelAffinityUsageCacheTestIDs(t *testing.T) (string, string, string) { + t.Helper() + + seq := channelAffinityUsageCacheTestSeq.Add(1) + safeName := strings.NewReplacer("/", "_", " ", "_").Replace(t.Name()) + ruleName := fmt.Sprintf("%s_rule_%d", safeName, seq) usingGroup := "default" - keyFP := fmt.Sprintf("fp_%d", time.Now().UnixNano()) + keyFP := fmt.Sprintf("%s_fp_%d", safeName, seq) + + entryKey := channelAffinityUsageCacheEntryKey(ruleName, usingGroup, keyFP) + cache := getChannelAffinityUsageCacheStatsCache() + _, _ = cache.DeleteMany([]string{entryKey}) + t.Cleanup(func() { + _, _ = cache.DeleteMany([]string{entryKey}) + }) + + return ruleName, usingGroup, keyFP +} + +func TestObserveChannelAffinityUsageCacheByRelayFormat_ClaudeMode(t *testing.T) { + ruleName, usingGroup, keyFP := newChannelAffinityUsageCacheTestIDs(t) ctx := buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP) usage := &dto.Usage{ @@ -53,9 +73,7 @@ func TestObserveChannelAffinityUsageCacheByRelayFormat_ClaudeMode(t *testing.T) } func TestObserveChannelAffinityUsageCacheByRelayFormat_MixedMode(t *testing.T) { - ruleName := fmt.Sprintf("rule_%d", time.Now().UnixNano()) - usingGroup := "default" - keyFP := fmt.Sprintf("fp_%d", time.Now().UnixNano()) + ruleName, usingGroup, keyFP := newChannelAffinityUsageCacheTestIDs(t) ctx := buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP) openAIUsage := &dto.Usage{ @@ -83,9 +101,7 @@ func TestObserveChannelAffinityUsageCacheByRelayFormat_MixedMode(t *testing.T) { } func TestObserveChannelAffinityUsageCacheByRelayFormat_UnsupportedModeKeepsEmpty(t *testing.T) { - ruleName := fmt.Sprintf("rule_%d", time.Now().UnixNano()) - usingGroup := "default" - keyFP := fmt.Sprintf("fp_%d", time.Now().UnixNano()) + ruleName, usingGroup, keyFP := newChannelAffinityUsageCacheTestIDs(t) ctx := buildChannelAffinityStatsContextForTest(ruleName, usingGroup, keyFP) usage := &dto.Usage{ diff --git a/service/governor/manager.go b/service/governor/manager.go new file mode 100644 index 000000000000..e245312986bc --- /dev/null +++ b/service/governor/manager.go @@ -0,0 +1,287 @@ +package governor + +import ( + "context" + "errors" + "net/http" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/model" + relayconstant "github.com/QuantumNous/new-api/relay/constant" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +var leaseHeartbeatStarter = startLeaseHeartbeat + +func PrepareAttemptForChannel(c *gin.Context, channel *model.Channel) (string, int, *types.NewAPIError) { + if channel == nil { + return "", 0, types.NewError(errors.New("channel is nil"), types.ErrorCodeGetChannelFailed, types.ErrOptionWithSkipRetry()) + } + + cfg := FromChannel(channel) + if !cfg.Enabled { + return channel.GetNextEnabledKey() + } + store := storeFactory() + if store == nil { + return channel.GetNextEnabledKey() + } + + ctx := requestContext(c) + if cooling, _, err := store.IsChannelCooling(ctx, channel.Id); err == nil && cooling { + return "", 0, selectionRejectedErr("channel is cooling") + } + + orderedIndices, apiErr := channel.OrderedEnabledKeyIndices() + if apiErr != nil { + return "", 0, apiErr + } + + applyKeyConcurrency := cfg.KeyMaxConcurrency > 0 && !isAsyncTaskSubmit(c) + for _, keyIndex := range orderedIndices { + if cooling, _, err := store.IsKeyCooling(ctx, channel.Id, keyIndex); err == nil && cooling { + continue + } + + keyValue, keyErr := channel.KeyAt(keyIndex) + if keyErr != nil { + return "", 0, keyErr + } + + reservationID := "" + leaseHeld := false + var stopHeartbeat context.CancelFunc + attempt := &AttemptState{ + ChannelID: channel.Id, + ChannelName: channel.Name, + KeyIndex: keyIndex, + KeyValue: keyValue, + ApplyKeyConcurrency: applyKeyConcurrency, + Config: cfg, + } + if applyKeyConcurrency { + reservationID = uuid.NewString() + leaseTTL := time.Duration(cfg.ReservationLeaseSeconds) * time.Second + waitUntil := time.Now().Add(time.Duration(cfg.ShortWaitMS) * time.Millisecond) + for { + acquired, err := store.AcquireKeyLease(ctx, channel.Id, keyIndex, reservationID, cfg.KeyMaxConcurrency, leaseTTL) + if err != nil { + return "", 0, governorInfraErr(err) + } + if acquired { + leaseHeld = true + if cfg.ReservationHeartbeatSeconds > 0 { + stopHeartbeat = leaseHeartbeatStarter( + ctx, + store, + channel.Id, + keyIndex, + reservationID, + leaseTTL, + time.Duration(cfg.ReservationHeartbeatSeconds)*time.Second, + ) + } + break + } + if cfg.ShortWaitMS <= 0 || time.Now().After(waitUntil) { + break + } + time.Sleep(25 * time.Millisecond) + } + if !leaseHeld { + continue + } + } + attempt.ReservationID = reservationID + attempt.LeaseHeld = leaseHeld + attempt.StopHeartbeat = stopHeartbeat + + if cfg.ChannelMaxRPM > 0 { + allowed, err := store.AllowChannelRPM(ctx, channel.Id, cfg.ChannelMaxRPM) + if err != nil { + releasePreparedAttempt(ctx, store, attempt) + return "", 0, governorInfraErr(err) + } + if !allowed { + releasePreparedAttempt(ctx, store, attempt) + return "", 0, selectionRejectedErr("channel rpm limited") + } + } + + if apiErr = channel.CommitSelectedKeyIndex(keyIndex); apiErr != nil { + releasePreparedAttempt(ctx, store, attempt) + return "", 0, apiErr + } + + if c != nil { + common.SetContextKey(c, constant.ContextKeyGovernorAttempt, attempt) + } + return keyValue, keyIndex, nil + } + + return "", 0, selectionRejectedErr("no governor-eligible keys or channels remain") +} + +func startLeaseHeartbeat(parent context.Context, store Store, channelID int, keyIndex int, reservationID string, leaseTTL time.Duration, interval time.Duration) context.CancelFunc { + if store == nil || reservationID == "" { + return func() {} + } + if interval <= 0 { + interval = 20 * time.Second + } + if leaseTTL <= 0 { + leaseTTL = 90 * time.Second + } + + ctx, cancel := context.WithCancel(parent) + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _ = store.TouchKeyLease(context.Background(), channelID, keyIndex, reservationID, leaseTTL) + } + } + }() + return cancel +} + +func CompleteRelayAttemptFromContext(c *gin.Context, apiErr *types.NewAPIError) { + attempt := getAttemptFromContext(c) + if attempt == nil { + return + } + defer clearAttemptFromContext(c) + + stopAndRelease(c, attempt) + store := storeFactory() + if store == nil { + return + } + + decision := ClassifyRelayError(attempt.Config, apiErr) + if decision.CoolChannel && decision.TTL > 0 { + _ = store.CoolChannel(requestContext(c), attempt.ChannelID, decision.TTL) + } + if decision.CoolKey && decision.TTL > 0 { + _ = store.CoolKey(requestContext(c), attempt.ChannelID, attempt.KeyIndex, decision.TTL) + } +} + +func CompleteTaskAttemptFromContext(c *gin.Context, taskErr *dto.TaskError) { + attempt := getAttemptFromContext(c) + if attempt == nil { + return + } + defer clearAttemptFromContext(c) + + stopAndRelease(c, attempt) + store := storeFactory() + if store == nil { + return + } + + decision := ClassifyTaskError(attempt.Config, taskErr) + if decision.CoolChannel && decision.TTL > 0 { + _ = store.CoolChannel(requestContext(c), attempt.ChannelID, decision.TTL) + } +} + +func isAsyncTaskSubmit(c *gin.Context) bool { + if c == nil { + return false + } + if relayMode, ok := c.Get("relay_mode"); ok { + if mode, ok := relayMode.(int); ok { + switch mode { + case relayconstant.RelayModeSunoSubmit, + relayconstant.RelayModeVideoSubmit, + relayconstant.RelayModeMidjourneyImagine, + relayconstant.RelayModeMidjourneyDescribe, + relayconstant.RelayModeMidjourneyBlend, + relayconstant.RelayModeMidjourneyChange, + relayconstant.RelayModeMidjourneySimpleChange, + relayconstant.RelayModeMidjourneyAction, + relayconstant.RelayModeMidjourneyModal, + relayconstant.RelayModeMidjourneyShorten, + relayconstant.RelayModeSwapFace, + relayconstant.RelayModeMidjourneyUpload, + relayconstant.RelayModeMidjourneyVideo, + relayconstant.RelayModeMidjourneyEdits: + return true + } + } + } + return false +} + +func selectionRejectedErr(message string) *types.NewAPIError { + return types.NewErrorWithStatusCode( + errors.New(message), + types.ErrorCodeGovernorSelectionRejected, + http.StatusTooManyRequests, + types.ErrOptionWithSkipRetry(), + types.ErrOptionWithNoRecordErrorLog(), + ) +} + +func governorInfraErr(err error) *types.NewAPIError { + return types.NewErrorWithStatusCode(err, types.ErrorCodeGetChannelFailed, http.StatusServiceUnavailable) +} + +func getAttemptFromContext(c *gin.Context) *AttemptState { + if c == nil { + return nil + } + v, ok := c.Get(string(constant.ContextKeyGovernorAttempt)) + if !ok || v == nil { + return nil + } + attempt, _ := v.(*AttemptState) + return attempt +} + +func clearAttemptFromContext(c *gin.Context) { + if c == nil || c.Keys == nil { + return + } + delete(c.Keys, string(constant.ContextKeyGovernorAttempt)) +} + +func stopAndRelease(c *gin.Context, attempt *AttemptState) { + if attempt == nil { + return + } + store := storeFactory() + releasePreparedAttempt(requestContext(c), store, attempt) +} + +func releasePreparedAttempt(ctx context.Context, store Store, attempt *AttemptState) { + if attempt == nil { + return + } + if attempt.StopHeartbeat != nil { + attempt.StopHeartbeat() + attempt.StopHeartbeat = nil + } + if !attempt.LeaseHeld || attempt.ReservationID == "" || store == nil { + return + } + _ = store.ReleaseKeyLease(ctx, attempt.ChannelID, attempt.KeyIndex, attempt.ReservationID) + attempt.LeaseHeld = false +} + +func requestContext(c *gin.Context) context.Context { + if c != nil && c.Request != nil && c.Request.Context() != nil { + return c.Request.Context() + } + return context.Background() +} diff --git a/service/governor/manager_test.go b/service/governor/manager_test.go new file mode 100644 index 000000000000..8596d311399c --- /dev/null +++ b/service/governor/manager_test.go @@ -0,0 +1,235 @@ +package governor + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/model" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type managerTestStore struct { + acquireErr error + allowRPM bool + allowRPMErr error + releaseCalls []leaseCall + coolKeyCalls []coolCall + coolChCalls []coolCall +} + +type leaseCall struct { + channelID int + keyIndex int + reservationID string +} + +type coolCall struct { + channelID int + keyIndex int + ttl time.Duration +} + +func (s *managerTestStore) IsChannelCooling(_ context.Context, _ int) (bool, time.Duration, error) { + return false, 0, nil +} + +func (s *managerTestStore) IsKeyCooling(_ context.Context, _ int, _ int) (bool, time.Duration, error) { + return false, 0, nil +} + +func (s *managerTestStore) AllowChannelRPM(_ context.Context, _ int, _ int64) (bool, error) { + if s.allowRPMErr != nil { + return false, s.allowRPMErr + } + if s.allowRPM { + return true, nil + } + return true, nil +} + +func (s *managerTestStore) AcquireKeyLease(_ context.Context, _ int, _ int, _ string, _ int64, _ time.Duration) (bool, error) { + if s.acquireErr != nil { + return false, s.acquireErr + } + return true, nil +} + +func (s *managerTestStore) TouchKeyLease(_ context.Context, _ int, _ int, _ string, _ time.Duration) error { + return nil +} + +func (s *managerTestStore) ReleaseKeyLease(_ context.Context, channelID int, keyIndex int, reservationID string) error { + s.releaseCalls = append(s.releaseCalls, leaseCall{channelID: channelID, keyIndex: keyIndex, reservationID: reservationID}) + return nil +} + +func (s *managerTestStore) CoolChannel(_ context.Context, channelID int, ttl time.Duration) error { + s.coolChCalls = append(s.coolChCalls, coolCall{channelID: channelID, ttl: ttl}) + return nil +} + +func (s *managerTestStore) CoolKey(_ context.Context, channelID int, keyIndex int, ttl time.Duration) error { + s.coolKeyCalls = append(s.coolKeyCalls, coolCall{channelID: channelID, keyIndex: keyIndex, ttl: ttl}) + return nil +} + +func TestCompleteRelayAttemptFromContext_ReleasesLeaseAndCoolsKey(t *testing.T) { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + store := &managerTestStore{} + restoreStore := SetStoreFactoryForTest(func() Store { return store }) + defer restoreStore() + + common.SetContextKey(c, constant.ContextKeyGovernorAttempt, &AttemptState{ + ChannelID: 7, + KeyIndex: 1, + ReservationID: "lease-1", + LeaseHeld: true, + ApplyKeyConcurrency: true, + Config: Config{ + Enabled: true, + KeyCooldownSeconds: 11, + KeyCooldownOnStatuses: []int{429}, + RespectRetryAfter: true, + }, + }) + + apiErr := types.NewOpenAIError( + errors.New("rate limited"), + types.ErrorCodeBadResponseStatusCode, + http.StatusTooManyRequests, + types.ErrOptionWithRetryAfter("19"), + ) + CompleteRelayAttemptFromContext(c, apiErr) + + require.Len(t, store.releaseCalls, 1) + require.Equal(t, leaseCall{channelID: 7, keyIndex: 1, reservationID: "lease-1"}, store.releaseCalls[0]) + require.Len(t, store.coolKeyCalls, 1) + require.Equal(t, 7, store.coolKeyCalls[0].channelID) + require.Equal(t, 1, store.coolKeyCalls[0].keyIndex) + require.Equal(t, 19*time.Second, store.coolKeyCalls[0].ttl) + require.Len(t, store.coolChCalls, 0) + _, exists := c.Get(string(constant.ContextKeyGovernorAttempt)) + require.False(t, exists) +} + +func TestCompleteTaskAttemptFromContext_ReleasesLeaseButSkipsKeyConcurrencyCooling(t *testing.T) { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + store := &managerTestStore{} + restoreStore := SetStoreFactoryForTest(func() Store { return store }) + defer restoreStore() + + common.SetContextKey(c, constant.ContextKeyGovernorAttempt, &AttemptState{ + ChannelID: 9, + KeyIndex: 0, + ReservationID: "lease-task", + LeaseHeld: true, + ApplyKeyConcurrency: false, + Config: Config{ + Enabled: true, + ChannelCooldownSeconds: 25, + ChannelCooldownOnStatuses: []int{429}, + }, + }) + + taskErr := &dto.TaskError{ + Code: "fail_to_fetch_task", + Message: "rate limited", + StatusCode: http.StatusTooManyRequests, + Error: errors.New("rate limited"), + } + CompleteTaskAttemptFromContext(c, taskErr) + + require.Len(t, store.releaseCalls, 1) + require.Equal(t, leaseCall{channelID: 9, keyIndex: 0, reservationID: "lease-task"}, store.releaseCalls[0]) + require.Len(t, store.coolChCalls, 1) + require.Equal(t, 9, store.coolChCalls[0].channelID) + require.Equal(t, 25*time.Second, store.coolChCalls[0].ttl) + require.Len(t, store.coolKeyCalls, 0) + _, exists := c.Get(string(constant.ContextKeyGovernorAttempt)) + require.False(t, exists) +} + +func TestPrepareAttemptForChannel_RPMStoreErrorStopsHeartbeatAndReturnsInfraError(t *testing.T) { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + + store := &managerTestStore{allowRPMErr: errors.New("redis unavailable")} + restoreStore := SetStoreFactoryForTest(func() Store { return store }) + defer restoreStore() + + cancelCalls := 0 + previousStarter := leaseHeartbeatStarter + leaseHeartbeatStarter = func(parent context.Context, store Store, channelID int, keyIndex int, reservationID string, leaseTTL time.Duration, interval time.Duration) context.CancelFunc { + return func() { + cancelCalls++ + } + } + defer func() { + leaseHeartbeatStarter = previousStarter + }() + + channel := buildGovernorManagedChannelForTest() + _, _, apiErr := PrepareAttemptForChannel(c, channel) + + require.NotNil(t, apiErr) + require.Equal(t, types.ErrorCodeGetChannelFailed, apiErr.GetErrorCode()) + require.Equal(t, http.StatusServiceUnavailable, apiErr.StatusCode) + require.Equal(t, 1, cancelCalls) + require.Len(t, store.releaseCalls, 1) + require.False(t, getAttemptFromContext(c) != nil) +} + +func TestPrepareAttemptForChannel_AcquireLeaseErrorReturnsInfraError(t *testing.T) { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + + store := &managerTestStore{acquireErr: errors.New("redis unavailable")} + restoreStore := SetStoreFactoryForTest(func() Store { return store }) + defer restoreStore() + + channel := buildGovernorManagedChannelForTest() + _, _, apiErr := PrepareAttemptForChannel(c, channel) + + require.NotNil(t, apiErr) + require.Equal(t, types.ErrorCodeGetChannelFailed, apiErr.GetErrorCode()) + require.Equal(t, http.StatusServiceUnavailable, apiErr.StatusCode) + require.Empty(t, store.releaseCalls) +} + +func buildGovernorManagedChannelForTest() *model.Channel { + channel := &model.Channel{ + Id: 21, + Name: "governed", + Key: "k1", + Status: common.ChannelStatusEnabled, + } + channel.SetSetting(dto.ChannelSettings{ + Governor: &dto.GovernorSettings{ + Enabled: true, + ChannelMaxRPM: 1, + KeyMaxConcurrency: 1, + ReservationLeaseSeconds: 30, + ReservationHeartbeatSeconds: 5, + }, + }) + return channel +} From 7fdf13e9878c5dd717360bab0c5203751e52ce6a Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 11 Apr 2026 12:32:30 +0800 Subject: [PATCH 15/16] Add fork-friendly lab validation tooling --- .github/workflows/fork-ghcr-image.yml | 93 ++++ .github/workflows/fork-verify.yml | 52 ++ .gitignore | 2 + deploy/lab/.env.example | 5 + .../channel-settings.governor.example.json | 15 + deploy/lab/cmd/governor-lab-bootstrap/main.go | 75 +++ deploy/lab/cmd/mock-openai-upstream/main.go | 35 ++ deploy/lab/compose.yml | 53 ++ deploy/lab/governorlab/bootstrap.go | 461 ++++++++++++++++++ deploy/lab/governorlab/bootstrap_test.go | 272 +++++++++++ deploy/lab/governorlab/mock.go | 200 ++++++++ deploy/lab/governorlab/mock_test.go | 126 +++++ deploy/lab/run-wsl-lab.sh | 76 +++ deploy/lab/run-wsl-mock-openai.sh | 34 ++ deploy/lab/start-wsl-lab.sh | 71 +++ deploy/lab/start-wsl-mock-openai.sh | 55 +++ deploy/lab/status-wsl-lab.sh | 112 +++++ deploy/lab/stop-wsl-lab.sh | 98 ++++ deploy/lab/test-stop-wsl-lab.sh | 82 ++++ deploy/lab/verify-governor.sh | 138 ++++++ deploy/lab/verify-wsl-governor-e2e.sh | 72 +++ docs/installation/fork-governor-lab.md | 251 ++++++++++ 22 files changed, 2378 insertions(+) create mode 100644 .github/workflows/fork-ghcr-image.yml create mode 100644 .github/workflows/fork-verify.yml create mode 100644 deploy/lab/.env.example create mode 100644 deploy/lab/channel-settings.governor.example.json create mode 100644 deploy/lab/cmd/governor-lab-bootstrap/main.go create mode 100644 deploy/lab/cmd/mock-openai-upstream/main.go create mode 100644 deploy/lab/compose.yml create mode 100644 deploy/lab/governorlab/bootstrap.go create mode 100644 deploy/lab/governorlab/bootstrap_test.go create mode 100644 deploy/lab/governorlab/mock.go create mode 100644 deploy/lab/governorlab/mock_test.go create mode 100644 deploy/lab/run-wsl-lab.sh create mode 100644 deploy/lab/run-wsl-mock-openai.sh create mode 100644 deploy/lab/start-wsl-lab.sh create mode 100644 deploy/lab/start-wsl-mock-openai.sh create mode 100644 deploy/lab/status-wsl-lab.sh create mode 100644 deploy/lab/stop-wsl-lab.sh create mode 100644 deploy/lab/test-stop-wsl-lab.sh create mode 100644 deploy/lab/verify-governor.sh create mode 100644 deploy/lab/verify-wsl-governor-e2e.sh create mode 100644 docs/installation/fork-governor-lab.md diff --git a/.github/workflows/fork-ghcr-image.yml b/.github/workflows/fork-ghcr-image.yml new file mode 100644 index 000000000000..9cee6e87d534 --- /dev/null +++ b/.github/workflows/fork-ghcr-image.yml @@ -0,0 +1,93 @@ +name: Fork Publish GHCR Image + +on: + push: + branches: + - main + - lab + workflow_dispatch: + inputs: + channel: + description: "Channel tag to publish, for example lab or main" + required: true + default: "lab" + type: string + push_latest: + description: "Also publish the latest tag" + required: true + default: false + type: boolean + +permissions: + contents: read + packages: write + +concurrency: + group: fork-ghcr-${{ github.ref }} + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - name: Check out + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve image metadata + id: meta + shell: bash + run: | + repo="ghcr.io/${GITHUB_REPOSITORY,,}" + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + channel="${{ github.event.inputs.channel }}" + else + channel="${GITHUB_REF_NAME}" + fi + + short_sha="${GITHUB_SHA::7}" + version="${channel}-$(date +'%Y%m%d')-${short_sha}" + + if [ ! -s VERSION ]; then + echo "${version}" > VERSION + fi + + echo "repo=${repo}" >> "${GITHUB_OUTPUT}" + echo "channel=${channel}" >> "${GITHUB_OUTPUT}" + echo "version=${version}" >> "${GITHUB_OUTPUT}" + + - name: Set up QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ steps.meta.outputs.repo }}:${{ steps.meta.outputs.channel }} + ${{ steps.meta.outputs.repo }}:${{ steps.meta.outputs.version }} + ${{ steps.meta.outputs.repo }}:sha-${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true + + - name: Publish latest tag + if: ${{ github.ref_name == 'main' || (github.event_name == 'workflow_dispatch' && github.event.inputs.push_latest == 'true') }} + run: | + docker buildx imagetools create \ + -t ${{ steps.meta.outputs.repo }}:latest \ + ${{ steps.meta.outputs.repo }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/fork-verify.yml b/.github/workflows/fork-verify.yml new file mode 100644 index 000000000000..c13ce68b3d9c --- /dev/null +++ b/.github/workflows/fork-verify.yml @@ -0,0 +1,52 @@ +name: Fork Verify + +on: + push: + pull_request: + +permissions: + contents: read + +concurrency: + group: fork-verify-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + + steps: + - name: Check out + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve build version + run: | + if [ ! -s VERSION ]; then + echo "dev-${GITHUB_SHA::7}" > VERSION + fi + + - name: Set up Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: latest + + - name: Set up Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version: "1.26.2" + + - name: Build frontend + env: + CI: "" + run: | + cd web + bun install + DISABLE_ESLINT_PLUGIN='true' VITE_REACT_APP_VERSION="$(cat ../VERSION)" bun run build + + - name: Run Go tests + run: | + go mod download + go test ./... -count=1 + + - name: Verify Docker build + run: docker build -t new-api-fork:verify . diff --git a/.gitignore b/.gitignore index 54fa83113f89..f8ce4709aeaa 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ data/ .gomodcache/ .gocache-temp .gopath +deploy/lab/.env +deploy/lab/runtime/ diff --git a/deploy/lab/.env.example b/deploy/lab/.env.example new file mode 100644 index 000000000000..a8bb2cbab82f --- /dev/null +++ b/deploy/lab/.env.example @@ -0,0 +1,5 @@ +NEW_API_IMAGE=ghcr.io/your-github-username/new-api:lab +HOST_HTTP_PORT=3000 +POSTGRES_PASSWORD=change-me +SESSION_SECRET=replace-this-before-sharing +TZ=Asia/Shanghai diff --git a/deploy/lab/channel-settings.governor.example.json b/deploy/lab/channel-settings.governor.example.json new file mode 100644 index 000000000000..8cea80ca836b --- /dev/null +++ b/deploy/lab/channel-settings.governor.example.json @@ -0,0 +1,15 @@ +{ + "governor": { + "enabled": true, + "channel_max_rpm": 120, + "channel_cooldown_seconds": 30, + "channel_cooldown_on_statuses": [429, 500, 502, 503, 504], + "key_max_concurrency": 4, + "key_cooldown_seconds": 20, + "key_cooldown_on_statuses": [429], + "reservation_lease_seconds": 90, + "reservation_heartbeat_seconds": 20, + "short_wait_ms": 150, + "respect_retry_after": true + } +} diff --git a/deploy/lab/cmd/governor-lab-bootstrap/main.go b/deploy/lab/cmd/governor-lab-bootstrap/main.go new file mode 100644 index 000000000000..ca9187ba7f8f --- /dev/null +++ b/deploy/lab/cmd/governor-lab-bootstrap/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "log" + "os" + + "github.com/QuantumNous/new-api/deploy/lab/governorlab" +) + +func main() { + baseURL := flag.String("base-url", "http://127.0.0.1:3000", "new-api base URL") + username := flag.String("username", "rootlab", "root username for lab setup/login") + password := flag.String("password", "rootpass123", "root password for lab setup/login") + selfUseMode := flag.Bool("self-use-mode", true, "enable self-use mode during setup") + demoSite := flag.Bool("demo-site", false, "enable demo site mode during setup") + channelName := flag.String("channel-name", "governor-lab-mock", "lab channel name") + channelKey := flag.String("channel-key", "mock-upstream-key", "upstream API key stored on the lab channel") + channelType := flag.Int("channel-type", 1, "channel type id") + channelModel := flag.String("channel-model", "gpt-4o-mini", "model name exposed by the lab channel") + channelGroup := flag.String("channel-group", "default", "channel group") + channelBaseURL := flag.String("channel-base-url", "http://127.0.0.1:8080", "mock upstream base URL") + channelSettingsFile := flag.String("channel-settings-file", "", "path to channel settings JSON") + tokenName := flag.String("token-name", "governor-lab-token", "user token name") + tokenGroup := flag.String("token-group", "default", "user token group") + outputEnvFile := flag.String("output-env-file", "", "optional env file path for shell wrappers") + flag.Parse() + + if *channelSettingsFile == "" { + log.Fatal("channel-settings-file is required") + } + + channelSettings, err := governorlab.LoadChannelSettings(*channelSettingsFile) + if err != nil { + log.Fatal(err) + } + + client, err := governorlab.NewClient(*baseURL) + if err != nil { + log.Fatal(err) + } + + result, err := client.Bootstrap(context.Background(), governorlab.BootstrapConfig{ + Username: *username, + Password: *password, + SelfUseModeEnabled: *selfUseMode, + DemoSiteEnabled: *demoSite, + ChannelName: *channelName, + ChannelKey: *channelKey, + ChannelType: *channelType, + ChannelModel: *channelModel, + ChannelGroup: *channelGroup, + ChannelBaseURL: *channelBaseURL, + ChannelSettings: channelSettings, + TokenName: *tokenName, + TokenGroup: *tokenGroup, + }) + if err != nil { + log.Fatal(err) + } + + if *outputEnvFile != "" { + if err := governorlab.WriteEnvFile(*outputEnvFile, result); err != nil { + log.Fatal(err) + } + } + + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(result); err != nil { + log.Fatal(err) + } +} diff --git a/deploy/lab/cmd/mock-openai-upstream/main.go b/deploy/lab/cmd/mock-openai-upstream/main.go new file mode 100644 index 000000000000..05d929aa28a4 --- /dev/null +++ b/deploy/lab/cmd/mock-openai-upstream/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "errors" + "flag" + "log" + "net/http" + "time" + + "github.com/QuantumNous/new-api/deploy/lab/governorlab" +) + +func main() { + listenAddr := flag.String("listen", ":8080", "listen address") + delayMS := flag.Int("delay-ms", 1500, "response delay in milliseconds") + responseText := flag.String("response-text", "ok", "mock assistant response text") + models := flag.String("models", "gpt-4o-mini", "comma-separated supported model list") + flag.Parse() + + handler := governorlab.NewMockHandler(governorlab.MockConfig{ + Delay: time.Duration(*delayMS) * time.Millisecond, + ResponseText: *responseText, + Models: governorlab.SplitCSV(*models), + }) + + server := &http.Server{ + Addr: *listenAddr, + Handler: handler, + } + + log.Printf("mock OpenAI upstream listening on %s (delay=%dms, models=%s)", *listenAddr, *delayMS, *models) + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatal(err) + } +} diff --git a/deploy/lab/compose.yml b/deploy/lab/compose.yml new file mode 100644 index 000000000000..78eddc3ad42f --- /dev/null +++ b/deploy/lab/compose.yml @@ -0,0 +1,53 @@ +name: new-api-lab + +services: + new-api: + image: ${NEW_API_IMAGE} + container_name: new-api-lab + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_started + ports: + - "${HOST_HTTP_PORT:-3000}:3000" + environment: + SQL_DSN: postgresql://root:${POSTGRES_PASSWORD}@postgres:5432/new-api?sslmode=disable + REDIS_CONN_STRING: redis://redis:6379/0 + SESSION_SECRET: ${SESSION_SECRET} + TZ: ${TZ:-Asia/Shanghai} + ERROR_LOG_ENABLED: "true" + BATCH_UPDATE_ENABLED: "true" + volumes: + - ./runtime/data:/data + - ./runtime/logs:/app/logs + healthcheck: + test: ["CMD-SHELL", "wget -q -O - http://localhost:3000/api/status | grep -o '\"success\":\\s*true' || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + + redis: + image: redis:7-alpine + container_name: new-api-lab-redis + restart: unless-stopped + command: ["redis-server", "--appendonly", "yes"] + volumes: + - ./runtime/redis:/data + + postgres: + image: postgres:16-alpine + container_name: new-api-lab-postgres + restart: unless-stopped + environment: + POSTGRES_USER: root + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: new-api + volumes: + - ./runtime/postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U root -d new-api"] + interval: 10s + timeout: 5s + retries: 5 diff --git a/deploy/lab/governorlab/bootstrap.go b/deploy/lab/governorlab/bootstrap.go new file mode 100644 index 000000000000..41fb3b052d80 --- /dev/null +++ b/deploy/lab/governorlab/bootstrap.go @@ -0,0 +1,461 @@ +package governorlab + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" +) + +type BootstrapConfig struct { + Username string + Password string + SelfUseModeEnabled bool + DemoSiteEnabled bool + ChannelName string + ChannelKey string + ChannelType int + ChannelModel string + ChannelGroup string + ChannelBaseURL string + ChannelSettings string + TokenName string + TokenGroup string +} + +type BootstrapResult struct { + UserID int `json:"user_id"` + ChannelID int `json:"channel_id"` + TokenID int `json:"token_id"` + APIKey string `json:"api_key"` + Model string `json:"model"` + BaseURL string `json:"base_url"` +} + +type Client struct { + baseURL string + httpClient *http.Client +} + +func NewClient(baseURL string) (*Client, error) { + jar, err := cookiejar.New(nil) + if err != nil { + return nil, err + } + return &Client{ + baseURL: strings.TrimRight(baseURL, "/"), + httpClient: &http.Client{ + Jar: jar, + }, + }, nil +} + +func RequiresPrivateIPAccess(baseURL string) bool { + parsed, err := url.Parse(baseURL) + if err != nil { + return false + } + + host := strings.ToLower(parsed.Hostname()) + if host == "" { + return false + } + if host == "localhost" { + return true + } + + ip := net.ParseIP(host) + if ip == nil { + return false + } + return ip.IsLoopback() || ip.IsPrivate() +} + +func (c *Client) Bootstrap(ctx context.Context, cfg BootstrapConfig) (*BootstrapResult, error) { + cfg = cfg.normalized() + if err := cfg.validate(); err != nil { + return nil, err + } + + setupState, err := c.getSetup(ctx) + if err != nil { + return nil, err + } + if !setupState.Status { + if err := c.postSetup(ctx, cfg); err != nil { + return nil, err + } + } + + userID, err := c.login(ctx, cfg.Username, cfg.Password) + if err != nil { + return nil, err + } + + if RequiresPrivateIPAccess(cfg.ChannelBaseURL) { + if err := c.updateOption(ctx, userID, "fetch_setting.allow_private_ip", true); err != nil { + return nil, err + } + } + + if err := c.recreateChannel(ctx, userID, cfg); err != nil { + return nil, err + } + if err := c.fixChannelAbilities(ctx, userID); err != nil { + return nil, err + } + + channelID, err := c.findChannelIDByName(ctx, userID, cfg.ChannelName) + if err != nil { + return nil, err + } + + if err := c.recreateToken(ctx, userID, cfg); err != nil { + return nil, err + } + + tokenID, err := c.findTokenIDByName(ctx, userID, cfg.TokenName) + if err != nil { + return nil, err + } + + apiKey, err := c.getTokenKey(ctx, userID, tokenID) + if err != nil { + return nil, err + } + + return &BootstrapResult{ + UserID: userID, + ChannelID: channelID, + TokenID: tokenID, + APIKey: apiKey, + Model: cfg.ChannelModel, + BaseURL: c.baseURL, + }, nil +} + +func WriteEnvFile(path string, result *BootstrapResult) error { + if result == nil { + return fmt.Errorf("bootstrap result is nil") + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + + lines := []string{ + "GOVERNOR_LAB_BASE_URL=" + quoteEnvValue(result.BaseURL), + "GOVERNOR_LAB_USER_ID=" + strconv.Itoa(result.UserID), + "GOVERNOR_LAB_CHANNEL_ID=" + strconv.Itoa(result.ChannelID), + "GOVERNOR_LAB_TOKEN_ID=" + strconv.Itoa(result.TokenID), + "GOVERNOR_LAB_API_KEY=" + quoteEnvValue(result.APIKey), + "GOVERNOR_LAB_MODEL=" + quoteEnvValue(result.Model), + } + content := strings.Join(lines, "\n") + "\n" + return os.WriteFile(path, []byte(content), 0o644) +} + +type apiEnvelope struct { + Success bool `json:"success"` + Message string `json:"message"` + Data json.RawMessage `json:"data"` +} + +type setupState struct { + Status bool `json:"status"` +} + +type loginState struct { + ID int `json:"id"` +} + +type channelSummary struct { + ID int `json:"id"` + Name string `json:"name"` +} + +type tokenSummary struct { + ID int `json:"id"` + Name string `json:"name"` +} + +func (cfg BootstrapConfig) normalized() BootstrapConfig { + if cfg.ChannelType == 0 { + cfg.ChannelType = 1 + } + if cfg.ChannelGroup == "" { + cfg.ChannelGroup = "default" + } + if cfg.TokenGroup == "" { + cfg.TokenGroup = "default" + } + cfg.ChannelBaseURL = strings.TrimRight(cfg.ChannelBaseURL, "/") + cfg.ChannelSettings = strings.TrimSpace(cfg.ChannelSettings) + return cfg +} + +func (cfg BootstrapConfig) validate() error { + switch { + case cfg.Username == "": + return fmt.Errorf("username is required") + case cfg.Password == "": + return fmt.Errorf("password is required") + case cfg.ChannelName == "": + return fmt.Errorf("channel name is required") + case cfg.ChannelKey == "": + return fmt.Errorf("channel key is required") + case cfg.ChannelModel == "": + return fmt.Errorf("channel model is required") + case cfg.ChannelBaseURL == "": + return fmt.Errorf("channel base URL is required") + case cfg.ChannelSettings == "": + return fmt.Errorf("channel settings JSON is required") + case cfg.TokenName == "": + return fmt.Errorf("token name is required") + } + return nil +} + +func (c *Client) getSetup(ctx context.Context) (*setupState, error) { + var state setupState + if err := c.doJSON(ctx, http.MethodGet, "/api/setup", 0, nil, &state); err != nil { + return nil, err + } + return &state, nil +} + +func (c *Client) postSetup(ctx context.Context, cfg BootstrapConfig) error { + payload := map[string]any{ + "username": cfg.Username, + "password": cfg.Password, + "confirmPassword": cfg.Password, + "SelfUseModeEnabled": cfg.SelfUseModeEnabled, + "DemoSiteEnabled": cfg.DemoSiteEnabled, + } + return c.doJSON(ctx, http.MethodPost, "/api/setup", 0, payload, nil) +} + +func (c *Client) login(ctx context.Context, username, password string) (int, error) { + var login loginState + if err := c.doJSON(ctx, http.MethodPost, "/api/user/login", 0, map[string]string{ + "username": username, + "password": password, + }, &login); err != nil { + return 0, err + } + if login.ID <= 0 { + return 0, fmt.Errorf("login did not return a valid user id") + } + return login.ID, nil +} + +func (c *Client) updateOption(ctx context.Context, userID int, key string, value any) error { + return c.doJSON(ctx, http.MethodPut, "/api/option/", userID, map[string]any{ + "key": key, + "value": value, + }, nil) +} + +func (c *Client) recreateChannel(ctx context.Context, userID int, cfg BootstrapConfig) error { + channels, err := c.listChannels(ctx, userID) + if err != nil { + return err + } + for _, entry := range channels { + if entry.Name == cfg.ChannelName { + if err := c.doJSON(ctx, http.MethodDelete, "/api/channel/"+strconv.Itoa(entry.ID), userID, nil, nil); err != nil { + return err + } + } + } + + payload := map[string]any{ + "mode": "single", + "channel": map[string]any{ + "name": cfg.ChannelName, + "type": cfg.ChannelType, + "key": cfg.ChannelKey, + "models": cfg.ChannelModel, + "group": cfg.ChannelGroup, + "base_url": cfg.ChannelBaseURL, + "setting": cfg.ChannelSettings, + "test_model": cfg.ChannelModel, + "status": 1, + }, + } + return c.doJSON(ctx, http.MethodPost, "/api/channel/", userID, payload, nil) +} + +func (c *Client) recreateToken(ctx context.Context, userID int, cfg BootstrapConfig) error { + tokens, err := c.listTokens(ctx, userID) + if err != nil { + return err + } + for _, entry := range tokens { + if entry.Name == cfg.TokenName { + if err := c.doJSON(ctx, http.MethodDelete, "/api/token/"+strconv.Itoa(entry.ID), userID, nil, nil); err != nil { + return err + } + } + } + + payload := map[string]any{ + "name": cfg.TokenName, + "remain_quota": 0, + "unlimited_quota": true, + "expired_time": -1, + "model_limits_enabled": false, + "group": cfg.TokenGroup, + } + return c.doJSON(ctx, http.MethodPost, "/api/token/", userID, payload, nil) +} + +func (c *Client) fixChannelAbilities(ctx context.Context, userID int) error { + return c.doJSON(ctx, http.MethodPost, "/api/channel/fix", userID, map[string]any{}, nil) +} + +func (c *Client) findChannelIDByName(ctx context.Context, userID int, name string) (int, error) { + channels, err := c.listChannels(ctx, userID) + if err != nil { + return 0, err + } + return highestNamedID(name, channels) +} + +func (c *Client) findTokenIDByName(ctx context.Context, userID int, name string) (int, error) { + tokens, err := c.listTokens(ctx, userID) + if err != nil { + return 0, err + } + return highestNamedID(name, tokens) +} + +func highestNamedID[T interface { + GetID() int + GetName() string +}](name string, entries []T) (int, error) { + matchID := 0 + for _, entry := range entries { + if entry.GetName() == name && entry.GetID() > matchID { + matchID = entry.GetID() + } + } + if matchID == 0 { + return 0, fmt.Errorf("no entry named %q found after creation", name) + } + return matchID, nil +} + +func (c *Client) listChannels(ctx context.Context, userID int) ([]channelSummary, error) { + var response struct { + Items []channelSummary `json:"items"` + } + if err := c.doJSON(ctx, http.MethodGet, "/api/channel/?p=1&page_size=1000&id_sort=true", userID, nil, &response); err != nil { + return nil, err + } + return response.Items, nil +} + +func (c *Client) listTokens(ctx context.Context, userID int) ([]tokenSummary, error) { + var response struct { + Items []tokenSummary `json:"items"` + } + if err := c.doJSON(ctx, http.MethodGet, "/api/token/?p=1&page_size=1000", userID, nil, &response); err != nil { + return nil, err + } + return response.Items, nil +} + +func (c *Client) getTokenKey(ctx context.Context, userID, tokenID int) (string, error) { + var response struct { + Key string `json:"key"` + } + if err := c.doJSON(ctx, http.MethodPost, "/api/token/"+strconv.Itoa(tokenID)+"/key", userID, map[string]any{}, &response); err != nil { + return "", err + } + if response.Key == "" { + return "", fmt.Errorf("token key response was empty") + } + return response.Key, nil +} + +func (c *Client) doJSON(ctx context.Context, method, path string, userID int, payload any, out any) error { + var body io.Reader + if payload != nil { + jsonBody, err := json.Marshal(payload) + if err != nil { + return err + } + body = bytes.NewReader(jsonBody) + } + + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if userID > 0 { + req.Header.Set("New-Api-User", strconv.Itoa(userID)) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + rawBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("request %s %s returned status %d: %s", method, path, resp.StatusCode, strings.TrimSpace(string(rawBody))) + } + + var envelope apiEnvelope + if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil { + return err + } + if !envelope.Success { + if envelope.Message == "" { + envelope.Message = "request failed" + } + return fmt.Errorf("%s %s failed: %s", method, path, envelope.Message) + } + if out == nil || len(envelope.Data) == 0 || bytes.Equal(bytes.TrimSpace(envelope.Data), []byte("null")) { + return nil + } + return json.Unmarshal(envelope.Data, out) +} + +func quoteEnvValue(value string) string { + replacer := strings.NewReplacer(`\`, `\\`, `"`, `\"`) + return `"` + replacer.Replace(value) + `"` +} + +func (entry channelSummary) GetID() int { return entry.ID } +func (entry channelSummary) GetName() string { return entry.Name } +func (entry tokenSummary) GetID() int { return entry.ID } +func (entry tokenSummary) GetName() string { return entry.Name } + +func LoadChannelSettings(path string) (string, error) { + content, err := os.ReadFile(path) + if err != nil { + return "", err + } + settings := strings.TrimSpace(string(content)) + if settings == "" { + return "", fmt.Errorf("channel settings file %s is empty", path) + } + return settings, nil +} + +func DefaultMockPortAllowed(port int) bool { + return slices.Contains([]int{80, 443, 8080, 8443}, port) +} diff --git a/deploy/lab/governorlab/bootstrap_test.go b/deploy/lab/governorlab/bootstrap_test.go new file mode 100644 index 000000000000..3dbf15e265ec --- /dev/null +++ b/deploy/lab/governorlab/bootstrap_test.go @@ -0,0 +1,272 @@ +package governorlab + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" +) + +func TestRequiresPrivateIPAccess(t *testing.T) { + t.Parallel() + + cases := []struct { + baseURL string + want bool + }{ + {baseURL: "http://127.0.0.1:8080", want: true}, + {baseURL: "http://localhost:8080", want: true}, + {baseURL: "http://172.22.240.24:8080", want: true}, + {baseURL: "https://api.openai.com", want: false}, + } + + for _, tc := range cases { + got := RequiresPrivateIPAccess(tc.baseURL) + if got != tc.want { + t.Fatalf("RequiresPrivateIPAccess(%q) = %v, want %v", tc.baseURL, got, tc.want) + } + } +} + +func TestBootstrapConfiguresLabViaHTTPAPI(t *testing.T) { + type channelEntry struct { + ID int `json:"id"` + Name string `json:"name"` + } + type tokenEntry struct { + ID int `json:"id"` + Name string `json:"name"` + } + + setupComplete := false + channels := []channelEntry{{ID: 7, Name: "governor-lab-mock"}} + tokens := []tokenEntry{{ID: 11, Name: "governor-lab-token"}} + nextChannelID := 8 + nextTokenID := 12 + optionUpdates := make(map[string]string) + var addedChannelPayload map[string]any + var addedTokenPayload map[string]any + fixChannelsCalls := 0 + + writeEnvelope := func(w http.ResponseWriter, data any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "success": true, + "message": "", + "data": data, + }) + } + + requireAuth := func(t *testing.T, w http.ResponseWriter, r *http.Request) bool { + t.Helper() + cookie, err := r.Cookie("session") + if err != nil || cookie.Value != "test-session" { + http.Error(w, `{"success":false,"message":"missing session"}`, http.StatusUnauthorized) + return false + } + if got := r.Header.Get("New-Api-User"); got != "1" { + http.Error(w, `{"success":false,"message":"missing New-Api-User"}`, http.StatusUnauthorized) + return false + } + return true + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/setup": + writeEnvelope(w, map[string]any{ + "status": setupComplete, + "root_init": false, + "database_type": "sqlite", + }) + case r.Method == http.MethodPost && r.URL.Path == "/api/setup": + setupComplete = true + writeEnvelope(w, nil) + case r.Method == http.MethodPost && r.URL.Path == "/api/user/login": + http.SetCookie(w, &http.Cookie{Name: "session", Value: "test-session", Path: "/"}) + writeEnvelope(w, map[string]any{ + "id": 1, + "username": "rootlab", + "display_name": "Root User", + "role": 100, + "status": 1, + "group": "default", + }) + case r.Method == http.MethodPut && r.URL.Path == "/api/option/": + if !requireAuth(t, w, r) { + return + } + var payload struct { + Key string `json:"key"` + Value any `json:"value"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatalf("expected valid option payload: %v", err) + } + optionUpdates[payload.Key] = stringifyValue(payload.Value) + writeEnvelope(w, nil) + case r.Method == http.MethodGet && r.URL.Path == "/api/channel/": + if !requireAuth(t, w, r) { + return + } + writeEnvelope(w, map[string]any{"items": channels}) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/api/channel/"): + if !requireAuth(t, w, r) { + return + } + idText := strings.TrimPrefix(r.URL.Path, "/api/channel/") + id, err := strconv.Atoi(idText) + if err != nil { + t.Fatalf("expected numeric channel id, got %q", idText) + } + filtered := make([]channelEntry, 0, len(channels)) + for _, entry := range channels { + if entry.ID != id { + filtered = append(filtered, entry) + } + } + channels = filtered + writeEnvelope(w, nil) + case r.Method == http.MethodPost && r.URL.Path == "/api/channel/": + if !requireAuth(t, w, r) { + return + } + if err := json.NewDecoder(r.Body).Decode(&addedChannelPayload); err != nil { + t.Fatalf("expected valid channel payload: %v", err) + } + channels = append(channels, channelEntry{ID: nextChannelID, Name: "governor-lab-mock"}) + nextChannelID++ + writeEnvelope(w, nil) + case r.Method == http.MethodPost && r.URL.Path == "/api/channel/fix": + if !requireAuth(t, w, r) { + return + } + fixChannelsCalls++ + writeEnvelope(w, map[string]any{"success": 1, "fails": 0}) + case r.Method == http.MethodGet && r.URL.Path == "/api/token/": + if !requireAuth(t, w, r) { + return + } + writeEnvelope(w, map[string]any{"items": tokens}) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/api/token/"): + if !requireAuth(t, w, r) { + return + } + idText := strings.TrimPrefix(r.URL.Path, "/api/token/") + id, err := strconv.Atoi(idText) + if err != nil { + t.Fatalf("expected numeric token id, got %q", idText) + } + filtered := make([]tokenEntry, 0, len(tokens)) + for _, entry := range tokens { + if entry.ID != id { + filtered = append(filtered, entry) + } + } + tokens = filtered + writeEnvelope(w, nil) + case r.Method == http.MethodPost && r.URL.Path == "/api/token/": + if !requireAuth(t, w, r) { + return + } + if err := json.NewDecoder(r.Body).Decode(&addedTokenPayload); err != nil { + t.Fatalf("expected valid token payload: %v", err) + } + tokens = append(tokens, tokenEntry{ID: nextTokenID, Name: "governor-lab-token"}) + nextTokenID++ + writeEnvelope(w, nil) + case r.Method == http.MethodPost && r.URL.Path == "/api/token/12/key": + if !requireAuth(t, w, r) { + return + } + writeEnvelope(w, map[string]any{"key": "sk-test-token"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client, err := NewClient(server.URL) + if err != nil { + t.Fatalf("expected client initialization to succeed, got error: %v", err) + } + + result, err := client.Bootstrap(context.Background(), BootstrapConfig{ + Username: "rootlab", + Password: "rootpass123", + SelfUseModeEnabled: true, + DemoSiteEnabled: false, + ChannelName: "governor-lab-mock", + ChannelKey: "mock-upstream-key", + ChannelType: 1, + ChannelModel: "gpt-4o-mini", + ChannelGroup: "default", + ChannelBaseURL: "http://127.0.0.1:8080", + ChannelSettings: `{"governor":{"enabled":true,"key_max_concurrency":1}}`, + TokenName: "governor-lab-token", + TokenGroup: "default", + }) + if err != nil { + t.Fatalf("expected bootstrap to succeed, got error: %v", err) + } + + if result.UserID != 1 { + t.Fatalf("expected user id 1, got %d", result.UserID) + } + if result.ChannelID != 8 { + t.Fatalf("expected recreated channel id 8, got %d", result.ChannelID) + } + if result.TokenID != 12 { + t.Fatalf("expected recreated token id 12, got %d", result.TokenID) + } + if result.APIKey != "sk-test-token" { + t.Fatalf("expected full token key, got %q", result.APIKey) + } + if result.Model != "gpt-4o-mini" { + t.Fatalf("expected model gpt-4o-mini, got %q", result.Model) + } + if optionUpdates["fetch_setting.allow_private_ip"] != "true" { + t.Fatalf("expected allow_private_ip to be enabled for local mock, got %q", optionUpdates["fetch_setting.allow_private_ip"]) + } + + channel, ok := addedChannelPayload["channel"].(map[string]any) + if !ok { + t.Fatalf("expected wrapped channel payload, got %#v", addedChannelPayload) + } + if got := channel["base_url"]; got != "http://127.0.0.1:8080" { + t.Fatalf("expected mock base url to be forwarded, got %#v", got) + } + if got := channel["setting"]; got != `{"governor":{"enabled":true,"key_max_concurrency":1}}` { + t.Fatalf("expected governor settings to be forwarded, got %#v", got) + } + if got := channel["models"]; got != "gpt-4o-mini" { + t.Fatalf("expected model list to be forwarded, got %#v", got) + } + + if got := addedTokenPayload["name"]; got != "governor-lab-token" { + t.Fatalf("expected token name to be forwarded, got %#v", got) + } + if got := addedTokenPayload["group"]; got != "default" { + t.Fatalf("expected token group to be forwarded, got %#v", got) + } + if fixChannelsCalls != 1 { + t.Fatalf("expected bootstrap to rebuild channel abilities/cache once, got %d calls", fixChannelsCalls) + } +} + +func stringifyValue(value any) string { + switch typed := value.(type) { + case string: + return typed + case bool: + if typed { + return "true" + } + return "false" + default: + return "" + } +} diff --git a/deploy/lab/governorlab/mock.go b/deploy/lab/governorlab/mock.go new file mode 100644 index 000000000000..e97026a4eebd --- /dev/null +++ b/deploy/lab/governorlab/mock.go @@ -0,0 +1,200 @@ +package governorlab + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +type MockConfig struct { + Delay time.Duration + ResponseText string + DefaultModel string + Models []string +} + +func (cfg MockConfig) normalized() MockConfig { + if cfg.ResponseText == "" { + cfg.ResponseText = "ok" + } + if cfg.DefaultModel == "" { + cfg.DefaultModel = "gpt-4o-mini" + } + if len(cfg.Models) == 0 { + cfg.Models = []string{cfg.DefaultModel} + } + return cfg +} + +func NewMockHandler(cfg MockConfig) http.Handler { + cfg = cfg.normalized() + + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + writeMockJSON(w, http.StatusOK, map[string]any{ + "ok": true, + }) + }) + mux.HandleFunc("/v1/models", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeMockJSON(w, http.StatusMethodNotAllowed, map[string]any{ + "error": map[string]any{"message": "method not allowed"}, + }) + return + } + now := time.Now().Unix() + data := make([]map[string]any, 0, len(cfg.Models)) + for _, modelName := range cfg.Models { + data = append(data, map[string]any{ + "id": modelName, + "object": "model", + "created": now, + "owned_by": "governorlab", + }) + } + writeMockJSON(w, http.StatusOK, map[string]any{ + "object": "list", + "data": data, + }) + }) + mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeMockJSON(w, http.StatusMethodNotAllowed, map[string]any{ + "error": map[string]any{"message": "method not allowed"}, + }) + return + } + + var request struct { + Model string `json:"model"` + Stream bool `json:"stream"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + writeMockJSON(w, http.StatusBadRequest, map[string]any{ + "error": map[string]any{"message": "invalid JSON request body"}, + }) + return + } + + modelName := request.Model + if modelName == "" { + modelName = cfg.DefaultModel + } + if cfg.Delay > 0 { + time.Sleep(cfg.Delay) + } + + if request.Stream { + writeMockStreamResponse(w, modelName, cfg.ResponseText) + return + } + + writeMockJSON(w, http.StatusOK, buildMockChatCompletion(modelName, cfg.ResponseText)) + }) + + return mux +} + +func buildMockChatCompletion(modelName, responseText string) map[string]any { + now := time.Now().Unix() + return map[string]any{ + "id": fmt.Sprintf("chatcmpl-mock-%d", now), + "object": "chat.completion", + "created": now, + "model": modelName, + "choices": []map[string]any{ + { + "index": 0, + "message": map[string]any{ + "role": "assistant", + "content": responseText, + }, + "finish_reason": "stop", + }, + }, + "usage": map[string]int{ + "prompt_tokens": 8, + "completion_tokens": 4, + "total_tokens": 12, + }, + } +} + +func writeMockStreamResponse(w http.ResponseWriter, modelName, responseText string) { + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + + flusher, ok := w.(http.Flusher) + if !ok { + writeMockJSON(w, http.StatusInternalServerError, map[string]any{ + "error": map[string]any{"message": "streaming not supported"}, + }) + return + } + + now := time.Now().Unix() + chunks := []map[string]any{ + { + "id": fmt.Sprintf("chatcmpl-mock-%d", now), + "object": "chat.completion.chunk", + "created": now, + "model": modelName, + "choices": []map[string]any{ + { + "index": 0, + "delta": map[string]any{ + "role": "assistant", + "content": responseText, + }, + "finish_reason": nil, + }, + }, + }, + { + "id": fmt.Sprintf("chatcmpl-mock-%d", now), + "object": "chat.completion.chunk", + "created": now, + "model": modelName, + "choices": []map[string]any{ + { + "index": 0, + "delta": map[string]any{}, + "finish_reason": "stop", + }, + }, + }, + } + + for _, chunk := range chunks { + payload, _ := json.Marshal(chunk) + _, _ = fmt.Fprintf(w, "data: %s\n\n", payload) + flusher.Flush() + } + _, _ = fmt.Fprint(w, "data: [DONE]\n\n") + flusher.Flush() +} + +func writeMockJSON(w http.ResponseWriter, status int, payload any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) +} + +func SplitCSV(value string) []string { + if strings.TrimSpace(value) == "" { + return nil + } + parts := strings.Split(value, ",") + result := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + result = append(result, part) + } + } + return result +} diff --git a/deploy/lab/governorlab/mock_test.go b/deploy/lab/governorlab/mock_test.go new file mode 100644 index 000000000000..8555942698bd --- /dev/null +++ b/deploy/lab/governorlab/mock_test.go @@ -0,0 +1,126 @@ +package governorlab + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestMockHandlerReturnsChatCompletion(t *testing.T) { + handler := NewMockHandler(MockConfig{ + Delay: 0, + ResponseText: "mock-ok", + Models: []string{"gpt-4o-mini"}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"gpt-4o-mini","messages":[{"role":"user","content":"hi"}],"stream":false}`)) + req.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + + handler.ServeHTTP(recorder, req) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", recorder.Code) + } + + var payload struct { + Model string `json:"model"` + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil { + t.Fatalf("expected valid JSON response, got error: %v", err) + } + + if payload.Model != "gpt-4o-mini" { + t.Fatalf("expected echoed model, got %q", payload.Model) + } + if len(payload.Choices) != 1 { + t.Fatalf("expected one choice, got %d", len(payload.Choices)) + } + if payload.Choices[0].Message.Content != "mock-ok" { + t.Fatalf("expected response text mock-ok, got %q", payload.Choices[0].Message.Content) + } + if payload.Choices[0].FinishReason != "stop" { + t.Fatalf("expected finish reason stop, got %q", payload.Choices[0].FinishReason) + } + if payload.Usage.TotalTokens <= 0 || payload.Usage.PromptTokens <= 0 || payload.Usage.CompletionTokens <= 0 { + t.Fatalf("expected positive usage values, got %+v", payload.Usage) + } +} + +func TestMockHandlerReturnsModelList(t *testing.T) { + handler := NewMockHandler(MockConfig{ + Delay: 0, + Models: []string{"gpt-4o-mini", "gpt-4.1-mini"}, + }) + + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + recorder := httptest.NewRecorder() + + handler.ServeHTTP(recorder, req) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", recorder.Code) + } + + var payload struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil { + t.Fatalf("expected valid JSON response, got error: %v", err) + } + + if len(payload.Data) != 2 { + t.Fatalf("expected two models, got %d", len(payload.Data)) + } + if payload.Data[0].ID != "gpt-4o-mini" || payload.Data[1].ID != "gpt-4.1-mini" { + t.Fatalf("unexpected model list: %+v", payload.Data) + } +} + +func TestMockHandlerStreamsMinimalSSE(t *testing.T) { + handler := NewMockHandler(MockConfig{ + Delay: 0, + ResponseText: "stream-ok", + Models: []string{"gpt-4o-mini"}, + }) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"gpt-4o-mini","messages":[{"role":"user","content":"hi"}],"stream":true}`)) + req.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + + started := time.Now() + handler.ServeHTTP(recorder, req) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", recorder.Code) + } + if contentType := recorder.Header().Get("Content-Type"); !strings.Contains(contentType, "text/event-stream") { + t.Fatalf("expected SSE content type, got %q", contentType) + } + body := recorder.Body.String() + if !strings.Contains(body, "stream-ok") { + t.Fatalf("expected streamed content in body, got %q", body) + } + if !strings.Contains(body, "[DONE]") { + t.Fatalf("expected [DONE] marker in stream body, got %q", body) + } + if time.Since(started) < 0 { + t.Fatalf("unexpected negative duration") + } +} diff --git a/deploy/lab/run-wsl-lab.sh b/deploy/lab/run-wsl-lab.sh new file mode 100644 index 000000000000..dcfa1797c8fd --- /dev/null +++ b/deploy/lab/run-wsl-lab.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNTIME_DIR="${ROOT_DIR}/deploy/lab/runtime/wsl" +REDIS_DIR="${RUNTIME_DIR}/redis" +LOG_DIR="${RUNTIME_DIR}/logs" +BIN_PATH="${RUNTIME_DIR}/new-api" +REDIS_PORT="${REDIS_PORT:-6380}" +APP_PORT="${PORT:-3000}" +SESSION_SECRET="${SESSION_SECRET:-wsl-lab-secret-change-me}" +TZ_VALUE="${TZ:-Asia/Shanghai}" +SQLITE_FILE="${SQLITE_FILE:-${RUNTIME_DIR}/new-api.db}" +SQLITE_PATH_VALUE="${SQLITE_FILE}?_busy_timeout=30000" +REDIS_CONN_STRING_VALUE="${REDIS_CONN_STRING:-redis://127.0.0.1:${REDIS_PORT}/0}" +BUILD_VERSION="${VITE_REACT_APP_VERSION:-dev-wsl}" +NODE_OPTIONS_VALUE="${NODE_OPTIONS:---max-old-space-size=4096}" +REDIS_PID_FILE="${REDIS_DIR}/redis.pid" +REDIS_LOG_FILE="${LOG_DIR}/redis.log" + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Missing required command: $1" >&2 + exit 1 + fi +} + +for cmd in go bun redis-server; do + require_cmd "$cmd" +done + +mkdir -p "${RUNTIME_DIR}" "${REDIS_DIR}" "${LOG_DIR}" + +echo "Building frontend..." +pushd "${ROOT_DIR}/web" >/dev/null +if [[ ! -d node_modules ]]; then + bun install +fi +DISABLE_ESLINT_PLUGIN='true' NODE_OPTIONS="${NODE_OPTIONS_VALUE}" VITE_REACT_APP_VERSION="${BUILD_VERSION}" bun run build +popd >/dev/null + +echo "Building backend..." +pushd "${ROOT_DIR}" >/dev/null +go build -o "${BIN_PATH}" . +popd >/dev/null + +if [[ -f "${REDIS_PID_FILE}" ]] && kill -0 "$(cat "${REDIS_PID_FILE}")" 2>/dev/null; then + echo "Redis already running on port ${REDIS_PORT}" +else + echo "Starting Redis on port ${REDIS_PORT}..." + redis-server \ + --daemonize yes \ + --port "${REDIS_PORT}" \ + --dir "${REDIS_DIR}" \ + --appendonly yes \ + --pidfile "${REDIS_PID_FILE}" \ + --logfile "${REDIS_LOG_FILE}" +fi + +export PORT="${APP_PORT}" +export SQLITE_PATH="${SQLITE_PATH_VALUE}" +export REDIS_CONN_STRING="${REDIS_CONN_STRING_VALUE}" +export SESSION_SECRET="${SESSION_SECRET}" +export TZ="${TZ_VALUE}" +export ERROR_LOG_ENABLED="${ERROR_LOG_ENABLED:-true}" +export BATCH_UPDATE_ENABLED="${BATCH_UPDATE_ENABLED:-true}" + +echo "Starting new-api on port ${APP_PORT}..." +echo "SQLite: ${SQLITE_FILE}" +echo "Redis: ${REDIS_CONN_STRING}" +if command -v hostname >/dev/null 2>&1; then + echo "WSL IPs: $(hostname -I 2>/dev/null || true)" + echo "If Windows cannot reach localhost:${APP_PORT}, try the first WSL IP above." +fi + +exec "${BIN_PATH}" --log-dir "${LOG_DIR}" diff --git a/deploy/lab/run-wsl-mock-openai.sh b/deploy/lab/run-wsl-mock-openai.sh new file mode 100644 index 000000000000..4029cdcdb980 --- /dev/null +++ b/deploy/lab/run-wsl-mock-openai.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNTIME_DIR="${ROOT_DIR}/deploy/lab/runtime/wsl" +LOG_DIR="${RUNTIME_DIR}/logs" +BIN_PATH="${RUNTIME_DIR}/mock-openai-upstream" +MOCK_PORT="${MOCK_PORT:-8080}" +MOCK_DELAY_MS="${MOCK_DELAY_MS:-1500}" +MOCK_RESPONSE_TEXT="${MOCK_RESPONSE_TEXT:-ok}" +MOCK_MODELS="${MOCK_MODELS:-gpt-4o-mini}" + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Missing required command: $1" >&2 + exit 1 + fi +} + +require_cmd go + +mkdir -p "${RUNTIME_DIR}" "${LOG_DIR}" + +echo "Building mock OpenAI upstream..." +pushd "${ROOT_DIR}" >/dev/null +go build -o "${BIN_PATH}" ./deploy/lab/cmd/mock-openai-upstream +popd >/dev/null + +echo "Starting mock OpenAI upstream on port ${MOCK_PORT}..." +exec "${BIN_PATH}" \ + --listen ":${MOCK_PORT}" \ + --delay-ms "${MOCK_DELAY_MS}" \ + --response-text "${MOCK_RESPONSE_TEXT}" \ + --models "${MOCK_MODELS}" diff --git a/deploy/lab/start-wsl-lab.sh b/deploy/lab/start-wsl-lab.sh new file mode 100644 index 000000000000..07df23b96873 --- /dev/null +++ b/deploy/lab/start-wsl-lab.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNTIME_DIR="${ROOT_DIR}/deploy/lab/runtime/wsl" +LOG_FILE="${RUNTIME_DIR}/lab-start.log" +PID_FILE="${RUNTIME_DIR}/new-api.pid" +SESSION_FILE="${RUNTIME_DIR}/session.env" +START_TIMEOUT_SECONDS="${START_TIMEOUT_SECONDS:-300}" + +LAB_APP_PORT_DEFAULT=3000 +LAB_REDIS_PORT_DEFAULT=6380 + +if [[ -f "${SESSION_FILE}" ]]; then + # shellcheck disable=SC1090 + source "${SESSION_FILE}" +fi + +APP_PORT="${PORT:-${LAB_APP_PORT:-${APP_PORT:-${LAB_APP_PORT_DEFAULT}}}}" +REDIS_PORT="${REDIS_PORT:-${LAB_REDIS_PORT:-${REDIS_PORT:-${LAB_REDIS_PORT_DEFAULT}}}}" + +find_app_pid() { + pgrep -fo "${RUNTIME_DIR}/new-api --log-dir ${RUNTIME_DIR}/logs" || true +} + +mkdir -p "${RUNTIME_DIR}" + +if [[ -f "${PID_FILE}" ]]; then + existing_pid="$(tr -d '\r\n' < "${PID_FILE}")" + if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then + echo "new-api lab is already running with pid ${existing_pid}" + exit 0 + fi +fi + +cat > "${SESSION_FILE}" < "${LOG_FILE}" +nohup env PORT="${APP_PORT}" REDIS_PORT="${REDIS_PORT}" "${ROOT_DIR}/deploy/lab/run-wsl-lab.sh" > "${LOG_FILE}" 2>&1 & +app_pid=$! +echo "${app_pid}" > "${PID_FILE}" + +echo "Started WSL lab with pid ${app_pid}" +echo "Waiting for health check on http://127.0.0.1:${APP_PORT}/api/status ..." + +deadline=$((SECONDS + START_TIMEOUT_SECONDS)) +while (( SECONDS < deadline )); do + if curl -fsS "http://127.0.0.1:${APP_PORT}/api/status" >/dev/null 2>&1; then + resolved_pid="$(find_app_pid)" + if [[ -n "${resolved_pid}" ]]; then + echo "${resolved_pid}" > "${PID_FILE}" + fi + echo "WSL lab is healthy." + exit 0 + fi + + if ! kill -0 "${app_pid}" 2>/dev/null; then + echo "WSL lab exited before becoming healthy." >&2 + tail -n 80 "${LOG_FILE}" || true + exit 1 + fi + + sleep 2 +done + +echo "Timed out waiting for WSL lab health check." >&2 +tail -n 80 "${LOG_FILE}" || true +exit 1 diff --git a/deploy/lab/start-wsl-mock-openai.sh b/deploy/lab/start-wsl-mock-openai.sh new file mode 100644 index 000000000000..b9ff041c372a --- /dev/null +++ b/deploy/lab/start-wsl-mock-openai.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNTIME_DIR="${ROOT_DIR}/deploy/lab/runtime/wsl" +PID_FILE="${RUNTIME_DIR}/mock-openai.pid" +LOG_FILE="${RUNTIME_DIR}/mock-openai.log" +MOCK_PORT="${MOCK_PORT:-8080}" +START_TIMEOUT_SECONDS="${START_TIMEOUT_SECONDS:-60}" + +find_mock_pid() { + pgrep -fo "${RUNTIME_DIR}/mock-openai-upstream --listen :${MOCK_PORT}" || true +} + +mkdir -p "${RUNTIME_DIR}" + +if [[ -f "${PID_FILE}" ]]; then + existing_pid="$(tr -d '\r\n' < "${PID_FILE}")" + if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then + echo "mock OpenAI upstream is already running with pid ${existing_pid}" + exit 0 + fi +fi + +: > "${LOG_FILE}" +nohup env MOCK_PORT="${MOCK_PORT}" MOCK_DELAY_MS="${MOCK_DELAY_MS:-1500}" MOCK_RESPONSE_TEXT="${MOCK_RESPONSE_TEXT:-ok}" MOCK_MODELS="${MOCK_MODELS:-gpt-4o-mini}" "${ROOT_DIR}/deploy/lab/run-wsl-mock-openai.sh" > "${LOG_FILE}" 2>&1 & +mock_pid=$! +echo "${mock_pid}" > "${PID_FILE}" + +echo "Started mock OpenAI upstream with pid ${mock_pid}" +echo "Waiting for health check on http://127.0.0.1:${MOCK_PORT}/healthz ..." + +deadline=$((SECONDS + START_TIMEOUT_SECONDS)) +while (( SECONDS < deadline )); do + if curl -fsS "http://127.0.0.1:${MOCK_PORT}/healthz" >/dev/null 2>&1; then + resolved_pid="$(find_mock_pid)" + if [[ -n "${resolved_pid}" ]]; then + echo "${resolved_pid}" > "${PID_FILE}" + fi + echo "mock OpenAI upstream is healthy." + exit 0 + fi + + if ! kill -0 "${mock_pid}" 2>/dev/null; then + echo "mock OpenAI upstream exited before becoming healthy." >&2 + tail -n 80 "${LOG_FILE}" || true + exit 1 + fi + + sleep 1 +done + +echo "Timed out waiting for mock OpenAI upstream health check." >&2 +tail -n 80 "${LOG_FILE}" || true +exit 1 diff --git a/deploy/lab/status-wsl-lab.sh b/deploy/lab/status-wsl-lab.sh new file mode 100644 index 000000000000..651ac3ec4c2a --- /dev/null +++ b/deploy/lab/status-wsl-lab.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNTIME_DIR="${ROOT_DIR}/deploy/lab/runtime/wsl" +PID_FILE="${RUNTIME_DIR}/new-api.pid" +SESSION_FILE="${RUNTIME_DIR}/session.env" +REDIS_PID_FILE="${RUNTIME_DIR}/redis/redis.pid" +MOCK_PID_FILE="${RUNTIME_DIR}/mock-openai.pid" +LOG_FILE="${RUNTIME_DIR}/lab-start.log" +MOCK_LOG_FILE="${RUNTIME_DIR}/mock-openai.log" +APP_PORT_DEFAULT=3000 +REDIS_PORT_DEFAULT=6380 +MOCK_PORT_DEFAULT=8080 +ENV_APP_PORT="${PORT-}" +ENV_REDIS_PORT="${REDIS_PORT-}" +ENV_MOCK_PORT="${MOCK_PORT-}" + +find_app_pid() { + pgrep -fo "${RUNTIME_DIR}/new-api --log-dir ${RUNTIME_DIR}/logs" || true +} + +if [[ -f "${SESSION_FILE}" ]]; then + # shellcheck disable=SC1090 + source "${SESSION_FILE}" +fi + +APP_PORT="${ENV_APP_PORT:-${LAB_APP_PORT:-${APP_PORT_DEFAULT}}}" +REDIS_PORT="${ENV_REDIS_PORT:-${LAB_REDIS_PORT:-${REDIS_PORT_DEFAULT}}}" +MOCK_PORT="${ENV_MOCK_PORT:-${MOCK_PORT_DEFAULT}}" + +echo "WSL lab runtime: ${RUNTIME_DIR}" +echo "App port: ${APP_PORT}" +echo "Redis port: ${REDIS_PORT}" +echo "Mock port: ${MOCK_PORT}" +echo "Log file: ${LOG_FILE}" +echo "Mock log file: ${MOCK_LOG_FILE}" + +echo +echo "Process status:" + +if [[ -f "${PID_FILE}" ]]; then + app_pid="$(tr -d '\r\n' < "${PID_FILE}")" + if [[ -n "${app_pid}" ]] && kill -0 "${app_pid}" 2>/dev/null; then + echo " new-api: running (pid ${app_pid})" + else + detected_pid="$(find_app_pid)" + if [[ -n "${detected_pid}" ]]; then + echo "${detected_pid}" > "${PID_FILE}" + echo " new-api: running (pid ${detected_pid})" + else + echo " new-api: pid file exists but process is not running" + fi + fi +else + detected_pid="$(find_app_pid)" + if [[ -n "${detected_pid}" ]]; then + echo "${detected_pid}" > "${PID_FILE}" + echo " new-api: running (pid ${detected_pid})" + else + echo " new-api: not running" + fi +fi + +if [[ -f "${REDIS_PID_FILE}" ]]; then + redis_pid="$(tr -d '\r\n' < "${REDIS_PID_FILE}")" + if [[ -n "${redis_pid}" ]] && kill -0 "${redis_pid}" 2>/dev/null; then + echo " redis: running (pid ${redis_pid})" + else + echo " redis: pid file exists but process is not running" + fi +else + echo " redis: not running" +fi + +if [[ -f "${MOCK_PID_FILE}" ]]; then + mock_pid="$(tr -d '\r\n' < "${MOCK_PID_FILE}")" + if [[ -n "${mock_pid}" ]] && kill -0 "${mock_pid}" 2>/dev/null; then + echo " mock-openai: running (pid ${mock_pid})" + else + echo " mock-openai: pid file exists but process is not running" + fi +else + echo " mock-openai: not running" +fi + +echo +echo "Health checks:" + +if curl -fsS "http://127.0.0.1:${APP_PORT}/api/status" >/dev/null 2>&1; then + echo " api: healthy at http://127.0.0.1:${APP_PORT}/api/status" +else + echo " api: unavailable at http://127.0.0.1:${APP_PORT}/api/status" +fi + +if command -v redis-cli >/dev/null 2>&1 && redis-cli -p "${REDIS_PORT}" ping >/dev/null 2>&1; then + echo " redis: PONG on port ${REDIS_PORT}" +else + echo " redis: unavailable on port ${REDIS_PORT}" +fi + +if curl -fsS "http://127.0.0.1:${MOCK_PORT}/healthz" >/dev/null 2>&1; then + echo " mock-openai: healthy at http://127.0.0.1:${MOCK_PORT}/healthz" +else + echo " mock-openai: unavailable at http://127.0.0.1:${MOCK_PORT}/healthz" +fi + +if command -v hostname >/dev/null 2>&1; then + echo + echo "WSL IPs:" + hostname -I 2>/dev/null || true +fi diff --git a/deploy/lab/stop-wsl-lab.sh b/deploy/lab/stop-wsl-lab.sh new file mode 100644 index 000000000000..687f4a296903 --- /dev/null +++ b/deploy/lab/stop-wsl-lab.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNTIME_DIR="${ROOT_DIR}/deploy/lab/runtime/wsl" +PID_FILE="${RUNTIME_DIR}/new-api.pid" +SESSION_FILE="${RUNTIME_DIR}/session.env" +REDIS_PID_FILE="${RUNTIME_DIR}/redis/redis.pid" +MOCK_PID_FILE="${RUNTIME_DIR}/mock-openai.pid" +APP_PID_PATTERN="${RUNTIME_DIR}/new-api --log-dir ${RUNTIME_DIR}/logs" +REDIS_PORT_DEFAULT=6380 +REDIS_FALLBACK_ENABLED=false + +if [[ -f "${SESSION_FILE}" ]]; then + # shellcheck disable=SC1090 + source "${SESSION_FILE}" + REDIS_FALLBACK_ENABLED=true +fi + +if [[ -n "${REDIS_PORT:-}" ]]; then + REDIS_FALLBACK_ENABLED=true +fi + +if [[ "${REDIS_FALLBACK_ENABLED}" == true ]]; then + REDIS_PORT="${REDIS_PORT:-${LAB_REDIS_PORT:-${REDIS_PORT_DEFAULT}}}" +fi + +find_app_pid() { + pgrep -fo "${APP_PID_PATTERN}" || true +} + +find_redis_pid() { + pgrep -fo "redis-server .*:${REDIS_PORT}" || true +} + +stop_pid_file() { + local label="$1" + local pid_file="$2" + + if [[ ! -f "${pid_file}" ]]; then + echo "${label}: not running" + return 0 + fi + + local pid + pid="$(tr -d '\r\n' < "${pid_file}")" + if [[ -z "${pid}" ]]; then + rm -f "${pid_file}" + echo "${label}: empty pid file removed" + return 0 + fi + + if ! kill -0 "${pid}" 2>/dev/null; then + rm -f "${pid_file}" + echo "${label}: stale pid file removed" + return 0 + fi + + kill "${pid}" 2>/dev/null || true + for _ in $(seq 1 20); do + if ! kill -0 "${pid}" 2>/dev/null; then + rm -f "${pid_file}" + echo "${label}: stopped" + return 0 + fi + sleep 1 + done + + kill -9 "${pid}" 2>/dev/null || true + rm -f "${pid_file}" + echo "${label}: force stopped" +} + +stop_pid_file "new-api" "${PID_FILE}" +stop_pid_file "redis" "${REDIS_PID_FILE}" +stop_pid_file "mock-openai" "${MOCK_PID_FILE}" + +fallback_app_pid="$(find_app_pid)" +if [[ -n "${fallback_app_pid}" ]]; then + kill "${fallback_app_pid}" 2>/dev/null || true + echo "new-api: stopped fallback pid ${fallback_app_pid}" +fi + +if [[ "${REDIS_FALLBACK_ENABLED}" == true ]]; then + fallback_redis_pid="$(find_redis_pid)" + if [[ -n "${fallback_redis_pid}" ]]; then + kill "${fallback_redis_pid}" 2>/dev/null || true + for _ in $(seq 1 20); do + if ! kill -0 "${fallback_redis_pid}" 2>/dev/null; then + echo "redis: stopped fallback pid ${fallback_redis_pid}" + exit 0 + fi + sleep 1 + done + kill -9 "${fallback_redis_pid}" 2>/dev/null || true + echo "redis: force stopped fallback pid ${fallback_redis_pid}" + fi +fi diff --git a/deploy/lab/test-stop-wsl-lab.sh b/deploy/lab/test-stop-wsl-lab.sh new file mode 100644 index 000000000000..ea806357c293 --- /dev/null +++ b/deploy/lab/test-stop-wsl-lab.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SOURCE_SCRIPT="${ROOT_DIR}/deploy/lab/stop-wsl-lab.sh" +TMP_DIR="$(mktemp -d)" +TEST_ROOT="${TMP_DIR}/workspace" +RUNTIME_DIR="${TEST_ROOT}/deploy/lab/runtime/wsl" +ORPHAN_PID_FILE="${TMP_DIR}/orphan-redis.pid" +ORPHAN_LOG_FILE="${TMP_DIR}/orphan-redis.log" +ORPHAN_DATA_DIR="${TMP_DIR}/orphan-redis" +OUTPUT_FILE="${TMP_DIR}/stop-output.txt" + +cleanup() { + if [[ -f "${ORPHAN_PID_FILE}" ]]; then + orphan_pid="$(tr -d '\r\n' < "${ORPHAN_PID_FILE}")" + if [[ -n "${orphan_pid}" ]] && kill -0 "${orphan_pid}" 2>/dev/null; then + kill "${orphan_pid}" 2>/dev/null || true + sleep 1 + kill -9 "${orphan_pid}" 2>/dev/null || true + fi + fi + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +find_free_port() { + local port + while true; do + port="$((20000 + RANDOM % 20000))" + if ! ss -ltn "sport = :${port}" | grep -q ":${port}"; then + echo "${port}" + return 0 + fi + done +} + +wait_for_redis() { + local port="$1" + for _ in $(seq 1 20); do + if redis-cli -p "${port}" PING >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +REDIS_PORT="$(find_free_port)" + +mkdir -p "${TEST_ROOT}/deploy/lab" "${RUNTIME_DIR}/redis" "${ORPHAN_DATA_DIR}" +cp "${SOURCE_SCRIPT}" "${TEST_ROOT}/deploy/lab/stop-wsl-lab.sh" +chmod +x "${TEST_ROOT}/deploy/lab/stop-wsl-lab.sh" + +cat > "${RUNTIME_DIR}/session.env" <&2 + exit 1 +fi + +"${TEST_ROOT}/deploy/lab/stop-wsl-lab.sh" >"${OUTPUT_FILE}" 2>&1 + +if redis-cli -p "${REDIS_PORT}" PING >/dev/null 2>&1; then + echo "expected stop-wsl-lab.sh to stop fallback redis on port ${REDIS_PORT}, but redis is still running" >&2 + cat "${OUTPUT_FILE}" >&2 + exit 1 +fi + +echo "stop-wsl-lab fallback redis cleanup passed on port ${REDIS_PORT}" diff --git a/deploy/lab/verify-governor.sh b/deploy/lab/verify-governor.sh new file mode 100644 index 000000000000..bf43d517d3d8 --- /dev/null +++ b/deploy/lab/verify-governor.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:3000}" +API_PATH="${API_PATH:-/v1/chat/completions}" +API_KEY="${API_KEY:-}" +MODEL="${MODEL:-}" +TOTAL_REQUESTS="${TOTAL_REQUESTS:-20}" +CONCURRENCY="${CONCURRENCY:-5}" +PROMPT="${PROMPT:-Reply with ok.}" +REQUEST_BODY_FILE="${REQUEST_BODY_FILE:-}" +EXPECT_HTTP_200_MIN="${EXPECT_HTTP_200_MIN:-}" +EXPECT_HTTP_429_MIN="${EXPECT_HTTP_429_MIN:-}" +EXPECT_GOVERNOR_REJECTIONS_MIN="${EXPECT_GOVERNOR_REJECTIONS_MIN:-}" + +if [[ -z "${API_KEY}" ]]; then + echo "API_KEY is required." >&2 + exit 1 +fi + +if [[ -z "${MODEL}" && -z "${REQUEST_BODY_FILE}" ]]; then + echo "MODEL is required when REQUEST_BODY_FILE is not provided." >&2 + exit 1 +fi + +if ! [[ "${TOTAL_REQUESTS}" =~ ^[0-9]+$ ]] || ! [[ "${CONCURRENCY}" =~ ^[0-9]+$ ]]; then + echo "TOTAL_REQUESTS and CONCURRENCY must be integers." >&2 + exit 1 +fi + +for expected_value in "${EXPECT_HTTP_200_MIN}" "${EXPECT_HTTP_429_MIN}" "${EXPECT_GOVERNOR_REJECTIONS_MIN}"; do + if [[ -n "${expected_value}" ]] && ! [[ "${expected_value}" =~ ^[0-9]+$ ]]; then + echo "Expectation values must be integers when provided." >&2 + exit 1 + fi +done + +if [[ "${TOTAL_REQUESTS}" -le 0 || "${CONCURRENCY}" -le 0 ]]; then + echo "TOTAL_REQUESTS and CONCURRENCY must be greater than zero." >&2 + exit 1 +fi + +tmp_dir="$(mktemp -d)" +payload_file="${tmp_dir}/payload.json" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +if [[ -n "${REQUEST_BODY_FILE}" ]]; then + cp "${REQUEST_BODY_FILE}" "${payload_file}" +else + cat >"${payload_file}" <"${code_file}" +} + +for index in $(seq 1 "${TOTAL_REQUESTS}"); do + run_one "${index}" & + while [[ "$(jobs -pr | wc -l | tr -d ' ')" -ge "${CONCURRENCY}" ]]; do + wait -n + done +done +wait + +declare -A counts=() +governor_rejections=0 + +for code_path in "${tmp_dir}"/*.code; do + code="$(cat "${code_path}")" + counts["${code}"]=$(( ${counts["${code}"]:-0} + 1 )) + + body_path="${code_path%.code}.body" + if grep -q 'governor:selection_rejected' "${body_path}"; then + governor_rejections=$((governor_rejections + 1)) + fi +done + +echo "Base URL: ${BASE_URL%/}${API_PATH}" +echo "Total requests: ${TOTAL_REQUESTS}" +echo "Configured concurrency: ${CONCURRENCY}" +echo +echo "HTTP status summary:" + +for code in "${!counts[@]}"; do + echo " HTTP ${code}: ${counts["${code}"]}" +done | sort + +echo +echo "governor:selection_rejected responses: ${governor_rejections}" + +sample_body="$(grep -l 'governor:selection_rejected' "${tmp_dir}"/*.body 2>/dev/null | head -n 1 || true)" +if [[ -n "${sample_body}" ]]; then + echo + echo "Sample governor rejection body:" + sed -n '1,20p' "${sample_body}" +fi + +get_count() { + local key="$1" + echo "${counts["${key}"]:-0}" +} + +expectation_failed=0 + +if [[ -n "${EXPECT_HTTP_200_MIN}" ]] && (( "$(get_count 200)" < EXPECT_HTTP_200_MIN )); then + echo "Expectation failed: expected at least ${EXPECT_HTTP_200_MIN} HTTP 200 responses." >&2 + expectation_failed=1 +fi + +if [[ -n "${EXPECT_HTTP_429_MIN}" ]] && (( "$(get_count 429)" < EXPECT_HTTP_429_MIN )); then + echo "Expectation failed: expected at least ${EXPECT_HTTP_429_MIN} HTTP 429 responses." >&2 + expectation_failed=1 +fi + +if [[ -n "${EXPECT_GOVERNOR_REJECTIONS_MIN}" ]] && (( governor_rejections < EXPECT_GOVERNOR_REJECTIONS_MIN )); then + echo "Expectation failed: expected at least ${EXPECT_GOVERNOR_REJECTIONS_MIN} governor:selection_rejected responses." >&2 + expectation_failed=1 +fi + +if (( expectation_failed != 0 )); then + exit 1 +fi diff --git a/deploy/lab/verify-wsl-governor-e2e.sh b/deploy/lab/verify-wsl-governor-e2e.sh new file mode 100644 index 000000000000..337d82be6040 --- /dev/null +++ b/deploy/lab/verify-wsl-governor-e2e.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNTIME_DIR="${ROOT_DIR}/deploy/lab/runtime/wsl" +SESSION_FILE="${RUNTIME_DIR}/session.env" +BOOTSTRAP_ENV_FILE="${RUNTIME_DIR}/governor-lab.env" +APP_PORT_DEFAULT=3000 +APP_PORT="${PORT:-}" + +if [[ -f "${SESSION_FILE}" ]]; then + # shellcheck disable=SC1090 + source "${SESSION_FILE}" +fi + +APP_PORT="${APP_PORT:-${LAB_APP_PORT:-${APP_PORT_DEFAULT}}}" +MOCK_PORT="${MOCK_PORT:-8080}" +MODEL="${MODEL:-gpt-4o-mini}" +LAB_ADMIN_USERNAME="${LAB_ADMIN_USERNAME:-rootlab}" +LAB_ADMIN_PASSWORD="${LAB_ADMIN_PASSWORD:-rootpass123}" +CHANNEL_NAME="${CHANNEL_NAME:-governor-lab-mock}" +TOKEN_NAME="${TOKEN_NAME:-governor-lab-token}" +CHANNEL_KEY="${CHANNEL_KEY:-mock-upstream-key}" +CHANNEL_GROUP="${CHANNEL_GROUP:-default}" +TOKEN_GROUP="${TOKEN_GROUP:-default}" +CHANNEL_SETTINGS_FILE="${CHANNEL_SETTINGS_FILE:-${ROOT_DIR}/deploy/lab/channel-settings.governor.example.json}" +TOTAL_REQUESTS="${TOTAL_REQUESTS:-12}" +CONCURRENCY="${CONCURRENCY:-4}" +PROMPT="${PROMPT:-Reply with ok.}" +EXPECT_HTTP_200_MIN="${EXPECT_HTTP_200_MIN:-1}" +EXPECT_HTTP_429_MIN="${EXPECT_HTTP_429_MIN:-1}" +EXPECT_GOVERNOR_REJECTIONS_MIN="${EXPECT_GOVERNOR_REJECTIONS_MIN:-1}" +BASE_URL="http://127.0.0.1:${APP_PORT}" +MOCK_BASE_URL="http://127.0.0.1:${MOCK_PORT}" + +if ! curl -fsS "${BASE_URL}/api/status" >/dev/null 2>&1; then + echo "new-api lab is not healthy at ${BASE_URL}/api/status" >&2 + echo "Start it first with deploy/lab/start-wsl-lab.sh" >&2 + exit 1 +fi + +"${ROOT_DIR}/deploy/lab/start-wsl-mock-openai.sh" + +pushd "${ROOT_DIR}" >/dev/null +go run ./deploy/lab/cmd/governor-lab-bootstrap \ + --base-url "${BASE_URL}" \ + --username "${LAB_ADMIN_USERNAME}" \ + --password "${LAB_ADMIN_PASSWORD}" \ + --channel-name "${CHANNEL_NAME}" \ + --channel-key "${CHANNEL_KEY}" \ + --channel-model "${MODEL}" \ + --channel-group "${CHANNEL_GROUP}" \ + --channel-base-url "${MOCK_BASE_URL}" \ + --channel-settings-file "${CHANNEL_SETTINGS_FILE}" \ + --token-name "${TOKEN_NAME}" \ + --token-group "${TOKEN_GROUP}" \ + --output-env-file "${BOOTSTRAP_ENV_FILE}" +popd >/dev/null + +# shellcheck disable=SC1090 +source "${BOOTSTRAP_ENV_FILE}" + +BASE_URL="${BASE_URL}" \ +API_KEY="${GOVERNOR_LAB_API_KEY}" \ +MODEL="${GOVERNOR_LAB_MODEL}" \ +TOTAL_REQUESTS="${TOTAL_REQUESTS}" \ +CONCURRENCY="${CONCURRENCY}" \ +PROMPT="${PROMPT}" \ +EXPECT_HTTP_200_MIN="${EXPECT_HTTP_200_MIN}" \ +EXPECT_HTTP_429_MIN="${EXPECT_HTTP_429_MIN}" \ +EXPECT_GOVERNOR_REJECTIONS_MIN="${EXPECT_GOVERNOR_REJECTIONS_MIN}" \ +"${ROOT_DIR}/deploy/lab/verify-governor.sh" diff --git a/docs/installation/fork-governor-lab.md b/docs/installation/fork-governor-lab.md new file mode 100644 index 000000000000..8013dd0a4c61 --- /dev/null +++ b/docs/installation/fork-governor-lab.md @@ -0,0 +1,251 @@ +# Fork + Governor 验证环境方案 + +这份文档用于把当前 governor 改动整理成一个可长期维护的 fork 工作流,并且提供一套独立的 `lab` 验证环境。目标不是继续扩大对 `new-api` 核心代码的改动,而是让后续验证、部署、回滚、升级都围绕镜像和环境来做。 + +本地验证优先推荐 `WSL`,Docker 不是本地验证的必须条件。更准确地说: + +- `WSL` 适合做日常开发、构建和 governor 行为验证。 +- Docker 更适合做可复制部署和 CI 制品发布。 +- governor 的分布式并发控制真正必须的运行时依赖是 `Redis`,不是 Docker。 + +## 推荐的仓库拓扑 + +推荐把你的 GitHub fork 设为 `origin`,官方仓库设为 `upstream`: + +```bash +git remote rename origin upstream +git remote add origin git@github.com:/new-api.git +git fetch --all --prune +``` + +推荐的分支职责: + +- `main`:fork 的稳定线,用于准备 canary 或更稳定的环境。 +- `lab`:用于快速验证 governor 与部署链,允许更频繁地发布镜像。 +- `feature/*`:短期功能分支,验证通过后并入 `lab`。 + +## 为什么这样更稳 + +- governor 的关键逻辑仍然留在应用内部,能感知渠道、key、重试与 `Retry-After`。 +- fork 额外新增的内容主要放在 `docs/`、`deploy/`、`.github/workflows/`,升级时冲突面更小。 +- 部署直接消费镜像,比每台机器 `git pull && 本地构建` 更稳定。 +- `lab` 和未来 `canary` 共享同一条交付链,只是流量级别不同。 + +## 环境分层 + +### `lab` + +用途: + +- 做合成流量验证。 +- 验证 Redis 下的 governor 行为。 +- 验证 `429`、冷却、并发占用释放、前后端构建链是否正常。 + +特点: + +- 不接正式生产流量。 +- 可以频繁替换镜像。 +- 优先验证“功能正确”和“部署可重复”。 +- 可以分成两种运行方式: + - `WSL` 原生运行:适合本地快速验证。 + - Docker Compose 运行:适合做接近部署形态的复现。 + +### `canary` + +用途: + +- 在 `lab` 通过后,承接一小部分真实流量。 +- 验证真实渠道限制、真实峰值行为、真实风控反馈。 + +特点: + +- 尽量复用 `lab` 已经验证过的镜像摘要。 +- 只调配置和流量比例,不重新构建产物。 + +## 推荐的交付模型 + +推荐顺序: + +1. 在 fork 分支上完成改动。 +2. 由 fork 的 GitHub Actions 跑验证工作流。 +3. 由 fork 的 GitHub Actions 构建并推送 GHCR 镜像。 +4. `lab` 环境拉取镜像并启动。 +5. 在 `lab` 中导入带 governor 的渠道配置并跑并发验证。 +6. 验证通过后,将同一个镜像摘要提升到 `canary`。 + +这样做的好处是:`lab` 和 `canary` 的差异主要来自配置与流量,而不是“重新构建了一次,结果产物已经变了”。 + +## 本地验证优先级 + +我现在更推荐这样分层: + +1. `WSL` 原生验证 +2. 远端 `lab` 环境验证 +3. `canary` 真实流量验证 +4. Docker 镜像验证与发布 + +其中第 1 层和第 2 层是功能验证主线,第 4 层更偏向交付和复现能力,不必把它误解成 governor 功能验证本身的前置条件。 + +## `lab` 环境目录 + +仓库里新增的 `deploy/lab` 包含: + +- `.env.example`:示例环境变量。 +- `compose.yml`:最小可运行的 `new-api + redis + postgres` 组合。 +- `channel-settings.governor.example.json`:可直接参考的渠道 governor 配置。 +- `verify-governor.sh`:合成并发验证脚本。 +- `start-wsl-lab.sh`:后台启动 WSL 本地 lab,并等待健康检查通过。 +- `status-wsl-lab.sh`:查看 WSL lab 的进程、端口和健康状态。 +- `stop-wsl-lab.sh`:停止 WSL lab 的 `new-api` 与 Redis 进程。 +- `run-wsl-lab.sh`:WSL 原生启动脚本,默认使用 `SQLite + Redis`。 +- `test-stop-wsl-lab.sh`:验证 `stop-wsl-lab.sh` 在 pid 文件缺失时仍能清理遗留 Redis 进程。 + +## WSL 原生验证 + +如果你的目标是先验证 governor,而不是先验证容器编排,那么 WSL 是更轻的路线。 + +推荐在 WSL 里单独 clone 一份仓库,而不是直接在 `/mnt/c/...` 下操作当前 Windows worktree: + +- Linux 侧 clone 的文件系统性能更稳。 +- Git 元数据路径更干净,不容易遇到 worktree 的 Windows 路径兼容问题。 +- 后续装依赖、跑构建、起服务都会更顺手。 + +WSL 最小依赖建议: + +- `go` +- `bun` +- `redis-server` +- `git` +- `curl` + +本地 `lab` 不一定需要 PostgreSQL。为了验证 governor,并不要求数据库必须是 PostgreSQL;使用 `SQLite + Redis` 就已经足够覆盖大部分并发治理逻辑。 + +启动方式: + +```bash +cd deploy/lab +./start-wsl-lab.sh +``` + +后台启动脚本会: + +- 启动后台 lab 进程 +- 等待 `/api/status` 健康检查通过 +- 把日志写到 `deploy/lab/runtime/wsl/lab-start.log` + +前台脚本 `run-wsl-lab.sh` 会: + +- 检查 `go`、`bun`、`redis-server` +- 构建前端 +- 构建后端 +- 启动一个本地 Redis +- 使用 `SQLite` 作为数据库启动 `new-api` + +常用命令: + +```bash +cd deploy/lab +./start-wsl-lab.sh +./status-wsl-lab.sh +./stop-wsl-lab.sh +``` + +如果你上一次异常退出,或者你手工删过 `deploy/lab/runtime/wsl` 里的文件,推荐先做一次干净重启: + +```bash +cd deploy/lab +./stop-wsl-lab.sh +rm -rf runtime/wsl +./start-wsl-lab.sh +``` + +现在的 `stop-wsl-lab.sh` 会优先按 pid 文件停进程;如果 Redis 的 pid 文件已经丢失,它还会按当前 lab 记录的 Redis 端口兜底清理遗留进程,避免下一次启动时误连到旧实例。 + +如果你的 Windows 宿主机无法通过 `http://127.0.0.1:` 访问 WSL 里的服务,不一定是 `new-api` 没起来,也可能只是当前 WSL 没启用本地端口回流。此时可以在 WSL 中执行: + +```bash +hostname -I +``` + +然后从 Windows 宿主访问第一个 WSL IP,例如: + +```text +http://:3000/api/status +``` + +启动步骤: + +```bash +cd deploy/lab +cp .env.example .env +docker compose -f compose.yml up -d +``` + +如果你使用的是另一台服务器,只需要: + +1. `git clone` 你的 fork。 +2. 修改 `deploy/lab/.env` 中的镜像地址与密码。 +3. `docker compose -f deploy/lab/compose.yml up -d`。 + +如果你在另一台 Linux 服务器上暂时不想用 Docker,也可以直接参考 `deploy/lab/run-wsl-lab.sh` 的环境变量方式,用 `systemd` 或 `supervisor` 管理二进制进程。 + +## Governor 配置建议 + +当前 governor 配置并不是环境变量,而是渠道 `settings` 里的 JSON。这样做反而更升级友好,因为不会再往全局配置层塞新的行为分支。 + +建议从 [deploy/lab/channel-settings.governor.example.json](../../deploy/lab/channel-settings.governor.example.json) 开始,先只给一小组测试渠道开启 governor,不要一开始全量铺开。 + +建议验证这些点: + +- 相同渠道在并发过高时是否出现 `governor:selection_rejected`。 +- `429` 是否表现为 governor 本地拒绝,而不是误判成泛化的 `503`。 +- 失败后的冷却期是否生效。 +- 请求结束后 lease 是否释放,后续请求能否恢复。 +- 多个相同优先级渠道是否会在 governor 拒绝后继续尝试后续候选项。 + +## 并发验证方式 + +在 `lab` 中至少准备: + +- 一个可用的测试 token。 +- 一个已绑定真实上游的测试模型。 +- 至少一个开启 governor 的渠道。 +- Redis 正常连接。 + +然后执行: + +```bash +BASE_URL=http://localhost:3000 \ +API_KEY= \ +MODEL= \ +TOTAL_REQUESTS=20 \ +CONCURRENCY=5 \ +./deploy/lab/verify-governor.sh +``` + +输出里重点看: + +- `HTTP 200` 的数量。 +- `HTTP 429` 的数量。 +- `governor:selection_rejected` 的出现次数。 + +如果你后续准备了真实流量环境,建议先在 `lab` 里把这套命令跑通,再把同一个镜像摘要推进到 `canary`。 + +## Upstream 升级节奏 + +推荐一个固定节奏: + +1. `git fetch upstream` +2. 将 `upstream/main` 合入 fork 的 `main` +3. 运行 fork 验证工作流 +4. 如需验证 governor,发布新的 `lab` 镜像 +5. `lab` 通过后再推进 `canary` + +这样未来如果 `new-api` 原生支持更好的并发治理,你只需要比较两套能力,再决定是否收缩 fork 差异,而不是先解决一堆混杂的部署脚本问题。 + +## 当前已知注意点 + +- governor 的分布式行为依赖 Redis;没有 Redis 时会退回默认选 key 行为。 +- 仓库中的 `VERSION` 文件当前为空,所以 fork workflow 里会在构建前写入一个临时版本号。 +- 对本地验证来说,Docker 不是必须条件。 +- 对跨机器稳定部署来说,镜像仍然是更稳的制品形态。 From 09967d68adfc3ffa9d3f6a29d1cec05c3aacce44 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 11 Apr 2026 12:33:05 +0800 Subject: [PATCH 16/16] Fix cache reentry for re-enabled channels --- model/channel_cache.go | 39 +++++++++++++++++++ service/channel_select_governor_test.go | 51 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/model/channel_cache.go b/model/channel_cache.go index 24da04e9c4ed..332a9913cda6 100644 --- a/model/channel_cache.go +++ b/model/channel_cache.go @@ -18,6 +18,33 @@ var group2model2channels map[string]map[string][]int // enabled channel var channelsIDM map[int]*Channel // all channels include disabled var channelSyncLock sync.RWMutex +func appendChannelToGroupModelCache(group, model string, channel *Channel) { + if group2model2channels == nil { + group2model2channels = make(map[string]map[string][]int) + } + if _, ok := group2model2channels[group]; !ok { + group2model2channels[group] = make(map[string][]int) + } + + channelIDs := group2model2channels[group][model] + for _, channelID := range channelIDs { + if channelID == channel.Id { + return + } + } + + channelIDs = append(channelIDs, channel.Id) + sort.Slice(channelIDs, func(i, j int) bool { + left := channelsIDM[channelIDs[i]] + right := channelsIDM[channelIDs[j]] + if left == nil || right == nil { + return false + } + return left.GetPriority() > right.GetPriority() + }) + group2model2channels[group][model] = channelIDs +} + func InitChannelCache() { if !common.MemoryCacheEnabled { return @@ -352,6 +379,18 @@ func CacheUpdateChannelStatus(id int, status int) { } } } + return + } + + channel, ok := channelsIDM[id] + if !ok || channel == nil { + return + } + + for _, group := range strings.Split(channel.Group, ",") { + for _, model := range strings.Split(channel.Models, ",") { + appendChannelToGroupModelCache(group, model, channel) + } } } diff --git a/service/channel_select_governor_test.go b/service/channel_select_governor_test.go index 1b903d053386..513b0d204a3a 100644 --- a/service/channel_select_governor_test.go +++ b/service/channel_select_governor_test.go @@ -115,3 +115,54 @@ func TestCacheGetRandomSatisfiedChannel_SkipsExcludedChannelsWithoutMemoryCache( } } +func TestCacheGetRandomSatisfiedChannel_ReenabledChannelReappearsInMemoryCache(t *testing.T) { + ensureGovernorSelectionSchema(t) + require.NoError(t, model.DB.Exec("DELETE FROM abilities").Error) + require.NoError(t, model.DB.Exec("DELETE FROM channels").Error) + t.Cleanup(func() { + model.DB.Exec("DELETE FROM abilities") + model.DB.Exec("DELETE FROM channels") + }) + + old := common.MemoryCacheEnabled + common.MemoryCacheEnabled = true + t.Cleanup(func() { common.MemoryCacheEnabled = old }) + + channelRecord := &model.Channel{ + Name: "reenable", + Key: "k-reenable", + Models: "gpt-4o", + Group: "default", + Status: common.ChannelStatusEnabled, + } + require.NoError(t, model.DB.Create(channelRecord).Error) + require.NoError(t, channelRecord.AddAbilities(nil)) + + model.InitChannelCache() + + selected, err := model.GetRandomSatisfiedChannel("default", "gpt-4o", 0) + require.NoError(t, err) + require.NotNil(t, selected) + require.Equal(t, channelRecord.Id, selected.Id) + + require.True(t, model.UpdateChannelStatus(channelRecord.Id, "", common.ChannelStatusAutoDisabled, "test disable")) + + disabled, err := model.GetRandomSatisfiedChannel("default", "gpt-4o", 0) + require.NoError(t, err) + require.Nil(t, disabled) + + require.True(t, model.UpdateChannelStatus(channelRecord.Id, "", common.ChannelStatusEnabled, "")) + + var dbChannel model.Channel + require.NoError(t, model.DB.First(&dbChannel, "id = ?", channelRecord.Id).Error) + require.Equal(t, common.ChannelStatusEnabled, dbChannel.Status) + + var dbAbility model.Ability + require.NoError(t, model.DB.First(&dbAbility, "channel_id = ?", channelRecord.Id).Error) + require.True(t, dbAbility.Enabled) + + reenabled, err := model.GetRandomSatisfiedChannel("default", "gpt-4o", 0) + require.NoError(t, err) + require.NotNil(t, reenabled) + require.Equal(t, channelRecord.Id, reenabled.Id) +}