Skip to content

Make Claude auto tests mirror Claude Code traffic #4228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
nixyme wants to merge 1 commit into from
Closed

Make Claude auto tests mirror Claude Code traffic #4228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 56 additions & 4 deletions controller/channel-test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,21 @@ import (
"github.com/gin-gonic/gin"
)

var claudeCodeTestHeaders = map[string]string{
"User-Agent": "claude-code/channel-test",
"X-Stainless-Arch": "arm64",
"X-Stainless-Lang": "js",
"X-Stainless-Os": "macos",
"X-Stainless-Package-Version": "channel-test",
"X-Stainless-Retry-Count": "0",
"X-Stainless-Runtime": "node",
"X-Stainless-Runtime-Version": "20.18.0",
"X-Stainless-Timeout": "600",
"X-App": "claude-code",
"Anthropic-Version": "2023-06-01",
"Anthropic-Dangerous-Direct-Browser-Access": "false",
}

type testResult struct {
context *gin.Context
localErr error
Expand All @@ -56,6 +71,32 @@ func normalizeChannelTestEndpoint(channel *model.Channel, modelName, endpointTyp
return normalized
}

func applyClaudeCodeTestFixtures(c *gin.Context, info *relaycommon.RelayInfo, request dto.Request) {
if c == nil || c.Request == nil || info == nil || info.RelayFormat != types.RelayFormatClaude {
return
}

for key, value := range claudeCodeTestHeaders {
if strings.TrimSpace(c.Request.Header.Get(key)) == "" {
c.Request.Header.Set(key, value)
}
}

if req, ok := request.(*dto.GeneralOpenAIRequest); ok && len(req.Metadata) == 0 {
req.Metadata = json.RawMessage(`{"user_id":"channel-test"}`)
}

runtimeHeaders := map[string]interface{}{}
for key, value := range relaycommon.GetEffectiveHeaderOverride(info) {
runtimeHeaders[key] = value
}
for key, value := range claudeCodeTestHeaders {
runtimeHeaders[key] = value
}
info.RuntimeHeadersOverride = runtimeHeaders
info.UseRuntimeHeadersOverride = true
}
Comment on lines +74 to +98

@coderabbitai coderabbitai Bot Apr 14, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Don't key the Claude fixture off inbound relay format alone.

testChannel still builds /v1/chat/completions by default, so Claude-backed channels often arrive here as RelayFormatOpenAI. Those requests are later converted to Claude upstreams in adaptors like relay/channel/aws/adaptor.go:124-128 and relay/channel/vertex/adaptor.go:327-336, so this guard skips the new headers/metadata for a chunk of the Claude traffic the PR is trying to harden. Gating on the resolved upstream Claude path would make this effective for the default test flow.

🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@controller/channel-test.go` around lines 74 - 98, The current guard in
applyClaudeCodeTestFixtures uses info.RelayFormat to detect Claude but misses
requests that start as OpenAI paths and are later adapted to Claude; change the
guard to detect the resolved upstream being Claude instead (inspect the
RelayInfo field that holds the resolved upstream target — e.g., info.Upstream,
info.UpstreamPath or info.UpstreamURL — and test for the Claude upstream
path/host used by your adaptors) and only return early when that resolved
upstream does not indicate Claude; keep the rest of applyClaudeCodeTestFixtures
(header injection, Metadata set, and RuntimeHeadersOverride) intact.


func testChannel(channel *model.Channel, testModel string, endpointType string, isStream bool) testResult {
tik := time.Now()
var unsupportedTestChannelTypes = []int{
Expand Down Expand Up @@ -231,6 +272,7 @@ func testChannel(channel *model.Channel, testModel string, endpointType string,

info.IsChannelTest = true
info.InitChannelMeta(c)
applyClaudeCodeTestFixtures(c, info, request)

err = helper.ModelMappedHelper(c, info, request)
if err != nil {
Expand Down Expand Up @@ -785,7 +827,7 @@ func TestChannel(c *gin.Context) {
var testAllChannelsLock sync.Mutex
var testAllChannelsRunning bool = false

func testAllChannels(notify bool) error {
func testAllChannels(notify bool, allowAutoDisable bool) error {

testAllChannelsLock.Lock()
if testAllChannelsRunning {
Expand Down Expand Up @@ -838,7 +880,17 @@ func testAllChannels(notify bool) error {

// disable channel
if isChannelEnabled && shouldBanChannel && channel.GetAutoBan() {
processChannelError(result.context, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError)
if allowAutoDisable {
processChannelError(result.context, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError)
} else {
common.SysLog(fmt.Sprintf(
"scheduled channel test detected unhealthy channel but skipped auto-disable: channel_id=%d name=%s status_code=%d reason=%s",
channel.Id,
channel.Name,
newAPIError.StatusCode,
newAPIError.Error(),
))
}
}

// enable channel
Expand All @@ -858,7 +910,7 @@ func testAllChannels(notify bool) error {
}

func TestAllChannels(c *gin.Context) {
err := testAllChannels(true)
err := testAllChannels(true, true)
if err != nil {
common.ApiError(c, err)
return
Expand Down Expand Up @@ -887,7 +939,7 @@ func AutomaticallyTestChannels() {
time.Sleep(time.Duration(int(math.Round(frequency))) * time.Minute)
common.SysLog(fmt.Sprintf("automatically test channels with interval %f minutes", frequency))
common.SysLog("automatically testing all channels")
_ = testAllChannels(false)
_ = testAllChannels(false, false)
common.SysLog("automatically channel test finished")
if !operation_setting.GetMonitorSetting().AutoTestChannelEnabled {
break
Expand Down
61 changes: 61 additions & 0 deletions controller/channel_test_fixture_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package controller

import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"

"github.com/QuantumNous/new-api/dto"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/types"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)

func TestApplyClaudeCodeTestFixturesInjectsHeadersAndMetadata(t *testing.T) {
gin.SetMode(gin.TestMode)

recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
ctx.Request.Header.Set("Content-Type", "application/json")

req := &dto.GeneralOpenAIRequest{
Model: "claude-sonnet-4-6",
Messages: []dto.Message{
{Role: "user", Content: "hi"},
},
}
info := &relaycommon.RelayInfo{
RelayFormat: types.RelayFormatClaude,
}

applyClaudeCodeTestFixtures(ctx, info, req)

require.JSONEq(t, `{"user_id":"channel-test"}`, string(req.Metadata))
require.Equal(t, "claude-code/channel-test", ctx.Request.Header.Get("User-Agent"))
require.Equal(t, "2023-06-01", ctx.Request.Header.Get("Anthropic-Version"))
require.True(t, info.UseRuntimeHeadersOverride)
require.Equal(t, "claude-code", info.RuntimeHeadersOverride["X-App"])
}

func TestApplyClaudeCodeTestFixturesPreservesExistingMetadata(t *testing.T) {
gin.SetMode(gin.TestMode)

recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)

req := &dto.GeneralOpenAIRequest{
Model: "claude-sonnet-4-6",
Metadata: json.RawMessage(`{"user_id":"existing-user"}`),
}
info := &relaycommon.RelayInfo{
RelayFormat: types.RelayFormatClaude,
}

applyClaudeCodeTestFixtures(ctx, info, req)

require.JSONEq(t, `{"user_id":"existing-user"}`, string(req.Metadata))
}
3 changes: 3 additions & 0 deletions relay/channel/claude/relay-claude.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ func RequestOpenAI2ClaudeMessage(c *gin.Context, textRequest dto.GeneralOpenAIRe
Temperature: textRequest.Temperature,
Tools: claudeTools,
}
if len(textRequest.Metadata) > 0 {
claudeRequest.Metadata = textRequest.Metadata
}
if maxTokens := textRequest.GetMaxTokens(); maxTokens > 0 {
claudeRequest.MaxTokens = common.GetPointer(maxTokens)
}
Expand Down
18 changes: 18 additions & 0 deletions relay/channel/claude/relay_claude_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package claude

import (
"encoding/base64"
"encoding/json"
"strings"
"testing"

Expand Down Expand Up @@ -363,3 +364,20 @@ func TestRequestOpenAI2ClaudeMessage_ConvertsTextFileContentToText(t *testing.T)
require.NotNil(t, content[0].Text)
require.Equal(t, "alpha\nbeta", *content[0].Text)
}

func TestRequestOpenAI2ClaudeMessage_PreservesMetadata(t *testing.T) {
request := dto.GeneralOpenAIRequest{
Model: "claude-3-5-sonnet",
Metadata: json.RawMessage(`{"user_id":"channel-test"}`),
Messages: []dto.Message{
{
Role: "user",
Content: "hi",
},
},
}

claudeRequest, err := RequestOpenAI2ClaudeMessage(nil, request)
require.NoError(t, err)
require.JSONEq(t, `{"user_id":"channel-test"}`, string(claudeRequest.Metadata))
}