diff --git a/controller/relay.go b/controller/relay.go index 1d14dcc6f880..aece2b25c6be 100644 --- a/controller/relay.go +++ b/controller/relay.go @@ -89,18 +89,18 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { defer func() { if newAPIError != nil { logger.LogError(c, fmt.Sprintf("relay error: %s", common.LocalLogPreview(newAPIError.Error()))) - newAPIError.SetMessage(common.MessageWithRequestId(newAPIError.Error(), requestId)) + responseError := types.ApplyDownstreamNewAPIErrorPolicy(newAPIError, requestId) switch relayFormat { case types.RelayFormatOpenAIRealtime: - helper.WssError(c, ws, newAPIError.ToOpenAIError()) + helper.WssError(c, ws, responseError.ToOpenAIError()) case types.RelayFormatClaude: - c.JSON(newAPIError.StatusCode, gin.H{ + c.JSON(responseError.StatusCode, gin.H{ "type": "error", - "error": newAPIError.ToClaudeError(), + "error": responseError.ToClaudeError(), }) default: - c.JSON(newAPIError.StatusCode, gin.H{ - "error": newAPIError.ToOpenAIError(), + c.JSON(responseError.StatusCode, gin.H{ + "error": responseError.ToOpenAIError(), }) } } @@ -378,6 +378,15 @@ func processChannelError(c *gin.Context, channelError types.ChannelError, err *t other["error_type"] = err.GetErrorType() other["error_code"] = err.GetErrorCode() other["status_code"] = err.StatusCode + if types.IsUpstreamError(err) { + other["upstream_error"] = true + other["client_status_code"] = http.StatusServiceUnavailable + if err.StatusCode != 0 { + other["upstream_status_code"] = err.StatusCode + } + } else { + other["upstream_error"] = false + } other["channel_id"] = channelId other["channel_name"] = c.GetString("channel_name") other["channel_type"] = c.GetInt("channel_type") diff --git a/docs/upstream-error-503-logmask.md b/docs/upstream-error-503-logmask.md new file mode 100644 index 000000000000..6a1f7e1d7920 --- /dev/null +++ b/docs/upstream-error-503-logmask.md @@ -0,0 +1,195 @@ +# Upstream Error 503 Policy + +This branch keeps upstream-provider failures private from downstream users. + +## Goal + +- Local New API errors stay unchanged. +- Upstream-origin errors returned to downstream clients become HTTP `503` with a generic service-unavailable message. +- User-visible logs also show upstream-origin errors as HTTP `503` with the same generic message. +- Admin/server logs and raw database records keep original details for troubleshooting. + +## Downstream Behavior + +When an upstream error happens, downstream OpenAI-compatible responses should look like: + +```json +{ + "error": { + "message": "Service temporarily unavailable. Please try again later. (request id: ...)", + "type": "new_api_error", + "code": "service_unavailable" + } +} +``` + +Local errors are not masked. For example, an invalid local token still returns: + +```json +{ + "error": { + "message": "Invalid token (request id: ...)", + "type": "new_api_error", + "code": "" + } +} +``` + +## User Log Behavior + +For ordinary user log APIs, upstream-origin error logs are masked: + +- `content` becomes `status_code=503, Service temporarily unavailable. Please try again later.` +- `other.status_code` becomes `503` +- `other.error_code`, `other.error_type`, `other.client_status_code`, and `other.upstream_status_code` are hidden +- channel fields and `upstream_request_id` are hidden from user-visible results + +Admin log APIs keep the original error details, including the real `status_code`; upstream errors also include +`client_status_code=503` and `upstream_status_code` for troubleshooting. + +## Main Files + +- `types/error.go` + - Adds `upstreamError` marker on `NewAPIError`. + - Adds `ApplyDownstreamNewAPIErrorPolicy`. + - Adds `MarkAsUpstreamError` and `IsUpstreamError`. + +- `controller/relay.go` + - Applies the downstream masking policy at the final response boundary. + - Records `other.upstream_error`, `other.client_status_code`, and `other.upstream_status_code` for error logs. + +- `service/error.go` + - Marks errors parsed from upstream non-2xx responses as upstream errors. + +- `model/log.go` + - Masks upstream-origin error logs only when formatting logs for ordinary users. + +- `relay/channel/*` + - Marks provider/body-level errors as upstream errors for channels that construct errors directly. + +- Tests: + - `types/error_policy_test.go` + - `model/log_test.go` + - `service/error_test.go` + +## Local Test Commands + +Use the local Go toolchain. If `proxy.golang.org` is slow, use `goproxy.cn`. + +```powershell +$env:GOPROXY='https://goproxy.cn,direct' +F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./types +F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./model -run "TestFormatUserLogs" +F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./service -run "Test(RelayErrorHandler|ResetStatusCode)" +F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./controller -run "^$" +``` + +Known unrelated failures seen in this source snapshot: + +- Full `./service` can fail in channel-affinity usage-cache tests. +- Full `./relay/channel/claude` can fail in existing file-content conversion tests. + +For compile-only checks: + +```powershell +$env:GOPROXY='https://goproxy.cn,direct' +F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./relay/channel/claude -run "^$" +``` + +## Frontend Build + +The backend embeds both frontend builds. Build them before building the Linux binary: + +```powershell +cd F:\workspace\mszb\.codex-tmp\new-api-src\web\default +$env:DISABLE_ESLINT_PLUGIN='true' +$env:VITE_REACT_APP_VERSION=(Get-Content ..\..\VERSION -Raw) +bun run build + +cd F:\workspace\mszb\.codex-tmp\new-api-src\web\classic +$env:VITE_REACT_APP_VERSION=(Get-Content ..\..\VERSION -Raw) +bun run build +``` + +## Local Linux Build + +Do not compile on the server. Build locally: + +```powershell +cd F:\workspace\mszb\.codex-tmp\new-api-src +$env:GOPROXY='https://goproxy.cn,direct' +$env:GOOS='linux' +$env:GOARCH='amd64' +$env:CGO_ENABLED='0' +$env:GOEXPERIMENT='greenteagc' +$version=''; if (Test-Path VERSION) { $raw=Get-Content VERSION -Raw; if ($null -ne $raw) { $version=$raw.Trim() } } +F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe build -ldflags "-s -w -X 'github.com/QuantumNous/new-api/common.Version=$version'" -o F:\workspace\mszb\.codex-tmp\new-api-linux-amd64 +``` + +## Deploy To 178 + +Upload and replace the binary. This does not compile on the server. + +```powershell +scp -i E:\MircDL\178.239.117.128_id_ed25519 -P 80 ` + F:\workspace\mszb\.codex-tmp\new-api-linux-amd64 ` + root@178.239.117.128:/opt/new-api/deploy/new-api-upstream-503-logmask + +$ts=(Get-Date -Format 'yyyyMMdd-HHmm') +ssh -i E:\MircDL\178.239.117.128_id_ed25519 -p 80 root@178.239.117.128 " +docker cp newapi:/new-api /opt/new-api/backup/new-api.bak-$ts && +docker cp /opt/new-api/deploy/new-api-upstream-503-logmask newapi:/tmp/new-api-upstream-503-logmask && +docker exec newapi sh -lc 'cp /new-api /new-api.bak-$ts && chmod +x /tmp/new-api-upstream-503-logmask && mv /tmp/new-api-upstream-503-logmask /new-api && sha256sum /new-api' && +docker restart newapi +" +``` + +Verify: + +```powershell +ssh -i E:\MircDL\178.239.117.128_id_ed25519 -p 80 root@178.239.117.128 ` + "docker ps --filter name=newapi --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}'" +``` + +Invalid local token should still return 401: + +```powershell +ssh -i E:\MircDL\178.239.117.128_id_ed25519 -p 80 root@178.239.117.128 " +curl -sS -i -X POST http://127.0.0.1:3001/v1/responses \ + -H 'Authorization: Bearer invalid-token-for-local-check' \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"gpt-5.5\",\"input\":\"hi\"}' | head -n 20 +" +``` + +## Rollback + +The container keeps timestamped binary backups: + +```powershell +ssh -i E:\MircDL\178.239.117.128_id_ed25519 -p 80 root@178.239.117.128 " +docker exec newapi sh -lc 'cp /new-api.bak-YYYYMMDD-HHMM /new-api && chmod +x /new-api' && +docker restart newapi +" +``` + +Host backups are stored in: + +```text +/opt/new-api/backup/ +``` + +## Updating From Upstream + +Recommended workflow: + +```powershell +cd F:\workspace\mszb\.codex-tmp\new-api-src +git fetch origin +git switch codex/upstream-error-503-logmask +git rebase origin/main +``` + +If conflicts happen, resolve them in the files listed in "Main Files", then rerun tests and build/deploy. + +Do not deploy the official `calciumion/new-api:latest` directly unless this branch has been rebuilt and redeployed, otherwise these masking changes will be lost. diff --git a/model/log.go b/model/log.go index 3db19f7e06e5..e3fccd19eba3 100644 --- a/model/log.go +++ b/model/log.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "strings" "time" @@ -72,6 +73,9 @@ func formatUserLogs(logs []*Log, startIdx int) { var otherMap map[string]interface{} otherMap, _ = common.StrToMap(logs[i].Other) if otherMap != nil { + if isUserVisibleUpstreamErrorLog(logs[i], otherMap) { + maskUserUpstreamErrorLog(logs[i], otherMap) + } // Remove admin-only debug fields. delete(otherMap, "admin_info") // delete(otherMap, "reject_reason") @@ -82,6 +86,48 @@ func formatUserLogs(logs []*Log, startIdx int) { } } +func isUserVisibleUpstreamErrorLog(log *Log, otherMap map[string]interface{}) bool { + if log == nil || log.Type != LogTypeError || otherMap == nil { + return false + } + if upstream, ok := otherMap["upstream_error"]; ok { + switch v := upstream.(type) { + case bool: + return v + case string: + return strings.EqualFold(v, "true") + } + } + code, _ := otherMap["error_code"].(string) + switch types.ErrorCode(code) { + case types.ErrorCodeDoRequestFailed, + types.ErrorCodeBadResponseStatusCode, + types.ErrorCodeBadResponse, + types.ErrorCodeReadResponseBodyFailed, + types.ErrorCodeBadResponseBody: + return true + default: + return false + } +} + +func maskUserUpstreamErrorLog(log *Log, otherMap map[string]interface{}) { + log.Content = fmt.Sprintf("status_code=%d, %s", http.StatusServiceUnavailable, types.PublicServiceUnavailableMessage) + log.ChannelId = 0 + log.ChannelName = "" + log.UpstreamRequestId = "" + + otherMap["status_code"] = http.StatusServiceUnavailable + delete(otherMap, "client_status_code") + delete(otherMap, "upstream_status_code") + delete(otherMap, "error_code") + delete(otherMap, "error_type") + delete(otherMap, "channel_id") + delete(otherMap, "channel_name") + delete(otherMap, "channel_type") + delete(otherMap, "upstream_error") +} + func GetLogByTokenId(tokenId int) (logs []*Log, err error) { err = LOG_DB.Model(&Log{}).Where("token_id = ?", tokenId).Order("id desc").Limit(common.MaxRecentItems).Find(&logs).Error formatUserLogs(logs, 0) diff --git a/model/log_test.go b/model/log_test.go new file mode 100644 index 000000000000..e2b075e0f943 --- /dev/null +++ b/model/log_test.go @@ -0,0 +1,99 @@ +package model + +import ( + "net/http" + "testing" + + "github.com/QuantumNous/new-api/common" +) + +func TestFormatUserLogsMasksUpstreamError(t *testing.T) { + logs := []*Log{{ + Id: 99, + Type: LogTypeError, + Content: "status_code=403, unexpected status 403 Forbidden: token quota is not enough, url: https://example.invalid/responses", + ChannelId: 12, + ChannelName: "upstream-channel", + UpstreamRequestId: "upstream-req-123", + Other: common.MapToJsonStr(map[string]interface{}{ + "status_code": http.StatusForbidden, + "client_status_code": http.StatusServiceUnavailable, + "upstream_status_code": http.StatusForbidden, + "error_code": "provider_quota_error", + "error_type": "openai_error", + "upstream_error": true, + "channel_id": 12, + "channel_name": "upstream-channel", + "channel_type": 1, + "admin_info": map[string]interface{}{ + "use_channel": []int{12}, + }, + }), + }} + + formatUserLogs(logs, 0) + + if logs[0].Content != "status_code=503, Service temporarily unavailable. Please try again later." { + t.Fatalf("content = %q", logs[0].Content) + } + if logs[0].ChannelId != 0 { + t.Fatalf("channel id = %d, want 0", logs[0].ChannelId) + } + if logs[0].ChannelName != "" { + t.Fatalf("channel name = %q, want empty", logs[0].ChannelName) + } + if logs[0].UpstreamRequestId != "" { + t.Fatalf("upstream request id = %q, want empty", logs[0].UpstreamRequestId) + } + + other, _ := common.StrToMap(logs[0].Other) + if other["status_code"] != float64(http.StatusServiceUnavailable) { + t.Fatalf("status code = %v, want %d", other["status_code"], http.StatusServiceUnavailable) + } + if _, ok := other["error_code"]; ok { + t.Fatal("error_code should be removed") + } + if _, ok := other["error_type"]; ok { + t.Fatal("error_type should be removed") + } + if _, ok := other["client_status_code"]; ok { + t.Fatal("client_status_code should be removed") + } + if _, ok := other["upstream_status_code"]; ok { + t.Fatal("upstream_status_code should be removed") + } + if _, ok := other["admin_info"]; ok { + t.Fatal("admin_info should be removed") + } + if _, ok := other["channel_id"]; ok { + t.Fatal("channel_id should be removed") + } + if _, ok := other["upstream_error"]; ok { + t.Fatal("upstream_error should be removed") + } +} + +func TestFormatUserLogsKeepsLocalError(t *testing.T) { + logs := []*Log{{ + Type: LogTypeError, + Content: "Invalid token", + ChannelId: 0, + UpstreamRequestId: "", + Other: common.MapToJsonStr(map[string]interface{}{ + "status_code": http.StatusUnauthorized, + "error_code": "", + "error_type": "new_api_error", + "upstream_error": false, + }), + }} + + formatUserLogs(logs, 0) + + if logs[0].Content != "Invalid token" { + t.Fatalf("content = %q", logs[0].Content) + } + other, _ := common.StrToMap(logs[0].Other) + if other["status_code"] != float64(http.StatusUnauthorized) { + t.Fatalf("status code = %v, want %d", other["status_code"], http.StatusUnauthorized) + } +} diff --git a/relay/channel/ali/image.go b/relay/channel/ali/image.go index a391e40ff4f1..fed43e0342c6 100644 --- a/relay/channel/ali/image.go +++ b/relay/channel/ali/image.go @@ -311,12 +311,12 @@ func aliImageHandler(a *Adaptor, c *gin.Context, resp *http.Response, info *rela return types.NewError(err, types.ErrorCodeBadResponse), nil } if aliResponse.Output.TaskStatus != "SUCCEEDED" { - return types.WithOpenAIError(types.OpenAIError{ + return types.MarkAsUpstreamError(types.WithOpenAIError(types.OpenAIError{ Message: aliResponse.Output.Message, Type: "ali_error", Param: "", Code: aliResponse.Output.Code, - }, resp.StatusCode), nil + }, resp.StatusCode)), nil } } diff --git a/relay/channel/ali/rerank.go b/relay/channel/ali/rerank.go index 1f7a3451fbac..3680b89dd116 100644 --- a/relay/channel/ali/rerank.go +++ b/relay/channel/ali/rerank.go @@ -46,12 +46,12 @@ func RerankHandler(c *gin.Context, resp *http.Response, info *relaycommon.RelayI } if aliResponse.Code != "" { - return types.WithOpenAIError(types.OpenAIError{ + return types.MarkAsUpstreamError(types.WithOpenAIError(types.OpenAIError{ Message: aliResponse.Message, Type: aliResponse.Code, Param: aliResponse.RequestId, Code: aliResponse.Code, - }, resp.StatusCode), nil + }, resp.StatusCode)), nil } usage := dto.Usage{ diff --git a/relay/channel/claude/relay-claude.go b/relay/channel/claude/relay-claude.go index 6356cd35796e..4d46f4279380 100644 --- a/relay/channel/claude/relay-claude.go +++ b/relay/channel/claude/relay-claude.go @@ -788,7 +788,7 @@ func HandleStreamResponseData(c *gin.Context, info *relaycommon.RelayInfo, claud return types.NewError(err, types.ErrorCodeBadResponseBody) } if claudeError := claudeResponse.GetClaudeError(); claudeError != nil && claudeError.Type != "" { - return types.WithClaudeError(*claudeError, http.StatusInternalServerError) + return types.MarkAsUpstreamError(types.WithClaudeError(*claudeError, http.StatusInternalServerError)) } if claudeResponse.StopReason != "" { maybeMarkClaudeRefusal(c, claudeResponse.StopReason) @@ -895,7 +895,7 @@ func HandleClaudeResponseData(c *gin.Context, info *relaycommon.RelayInfo, claud return types.NewError(err, types.ErrorCodeBadResponseBody) } if claudeError := claudeResponse.GetClaudeError(); claudeError != nil && claudeError.Type != "" { - return types.WithClaudeError(*claudeError, http.StatusInternalServerError) + return types.MarkAsUpstreamError(types.WithClaudeError(*claudeError, http.StatusInternalServerError)) } maybeMarkClaudeRefusal(c, claudeResponse.StopReason) if claudeInfo.Usage == nil { diff --git a/relay/channel/gemini/relay-gemini.go b/relay/channel/gemini/relay-gemini.go index e39826dd64e7..51dde5178004 100644 --- a/relay/channel/gemini/relay-gemini.go +++ b/relay/channel/gemini/relay-gemini.go @@ -1534,16 +1534,17 @@ func GeminiChatHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.R } service.ResetStatusCode(newAPIError, c.GetString("status_code_mapping")) + responseError := types.ApplyDownstreamNewAPIErrorPolicy(newAPIError, c.GetString(common.RequestIdKey)) switch info.RelayFormat { case types.RelayFormatClaude: - c.JSON(newAPIError.StatusCode, gin.H{ + c.JSON(responseError.StatusCode, gin.H{ "type": "error", - "error": newAPIError.ToClaudeError(), + "error": responseError.ToClaudeError(), }) default: - c.JSON(newAPIError.StatusCode, gin.H{ - "error": newAPIError.ToOpenAIError(), + c.JSON(responseError.StatusCode, gin.H{ + "error": responseError.ToOpenAIError(), }) } return &usage, nil diff --git a/relay/channel/jimeng/image.go b/relay/channel/jimeng/image.go index e422e061de6d..e03ec97f2673 100644 --- a/relay/channel/jimeng/image.go +++ b/relay/channel/jimeng/image.go @@ -64,12 +64,12 @@ func jimengImageHandler(c *gin.Context, resp *http.Response, info *relaycommon.R // Check if the response indicates an error if jimengResponse.Code != 10000 { - return nil, types.WithOpenAIError(types.OpenAIError{ + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(types.OpenAIError{ Message: jimengResponse.Message, Type: "jimeng_error", Param: "", Code: fmt.Sprintf("%d", jimengResponse.Code), - }, resp.StatusCode) + }, resp.StatusCode)) } // Convert Jimeng response to OpenAI format diff --git a/relay/channel/minimax/image.go b/relay/channel/minimax/image.go index 9b316bdc8230..53a1713ba500 100644 --- a/relay/channel/minimax/image.go +++ b/relay/channel/minimax/image.go @@ -187,11 +187,11 @@ func miniMaxImageHandler(c *gin.Context, resp *http.Response, info *relaycommon. return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } if minimaxResponse.BaseResp.StatusCode != 0 { - return nil, types.WithOpenAIError(types.OpenAIError{ + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(types.OpenAIError{ Message: minimaxResponse.BaseResp.StatusMsg, Type: "minimax_image_error", Code: fmt.Sprintf("%d", minimaxResponse.BaseResp.StatusCode), - }, resp.StatusCode) + }, resp.StatusCode)) } openAIResponse, err := responseMiniMax2OpenAIImage(&minimaxResponse, info) diff --git a/relay/channel/openai/chat_via_responses.go b/relay/channel/openai/chat_via_responses.go index 2c0752275daa..63c176ed17bb 100644 --- a/relay/channel/openai/chat_via_responses.go +++ b/relay/channel/openai/chat_via_responses.go @@ -56,7 +56,7 @@ func OaiResponsesToChatHandler(c *gin.Context, info *relaycommon.RelayInfo, resp } if oaiError := responsesResp.GetOpenAIError(); oaiError != nil && oaiError.Type != "" { - return nil, types.WithOpenAIError(*oaiError, resp.StatusCode) + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(*oaiError, resp.StatusCode)) } chatId := helper.GetResponseID(c) @@ -498,7 +498,7 @@ func OaiResponsesToChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo case "response.error", "response.failed": if streamResp.Response != nil { if oaiErr := streamResp.Response.GetOpenAIError(); oaiErr != nil && oaiErr.Type != "" { - streamErr = types.WithOpenAIError(*oaiErr, http.StatusInternalServerError) + streamErr = types.MarkAsUpstreamError(types.WithOpenAIError(*oaiErr, http.StatusInternalServerError)) sr.Stop(streamErr) return } diff --git a/relay/channel/openai/relay-openai.go b/relay/channel/openai/relay-openai.go index d6a354f71a22..f2261a775d9b 100644 --- a/relay/channel/openai/relay-openai.go +++ b/relay/channel/openai/relay-openai.go @@ -220,7 +220,7 @@ func OpenaiHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respo } if oaiError := simpleResponse.GetOpenAIError(); oaiError != nil && oaiError.Type != "" { - return nil, types.WithOpenAIError(*oaiError, resp.StatusCode) + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(*oaiError, resp.StatusCode)) } for _, choice := range simpleResponse.Choices { diff --git a/relay/channel/openai/relay_responses.go b/relay/channel/openai/relay_responses.go index 2665b8d027e9..7eadca1f195d 100644 --- a/relay/channel/openai/relay_responses.go +++ b/relay/channel/openai/relay_responses.go @@ -31,7 +31,7 @@ func OaiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } if oaiError := responsesResponse.GetOpenAIError(); oaiError != nil && oaiError.Type != "" { - return nil, types.WithOpenAIError(*oaiError, resp.StatusCode) + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(*oaiError, resp.StatusCode)) } if responsesResponse.HasImageGenerationCall() { diff --git a/relay/channel/openai/relay_responses_compact.go b/relay/channel/openai/relay_responses_compact.go index 390de8ed6865..3db1f7511171 100644 --- a/relay/channel/openai/relay_responses_compact.go +++ b/relay/channel/openai/relay_responses_compact.go @@ -25,7 +25,7 @@ func OaiResponsesCompactionHandler(c *gin.Context, resp *http.Response) (*dto.Us return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } if oaiError := compactResp.GetOpenAIError(); oaiError != nil && oaiError.Type != "" { - return nil, types.WithOpenAIError(*oaiError, resp.StatusCode) + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(*oaiError, resp.StatusCode)) } service.IOCopyBytesGracefully(c, resp, responseBody) diff --git a/relay/channel/palm/relay-palm.go b/relay/channel/palm/relay-palm.go index 786ea4cd2a20..a8360275fb5b 100644 --- a/relay/channel/palm/relay-palm.go +++ b/relay/channel/palm/relay-palm.go @@ -113,12 +113,12 @@ func palmHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respons return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } if palmResponse.Error.Code != 0 || len(palmResponse.Candidates) == 0 { - return nil, types.WithOpenAIError(types.OpenAIError{ + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(types.OpenAIError{ Message: palmResponse.Error.Message, Type: palmResponse.Error.Status, Param: "", Code: palmResponse.Error.Code, - }, resp.StatusCode) + }, resp.StatusCode)) } fullTextResponse := responsePaLM2OpenAI(&palmResponse) usage := service.ResponseText2Usage(c, palmResponse.Candidates[0].Content, info.UpstreamModelName, info.GetEstimatePromptTokens()) diff --git a/relay/channel/tencent/relay-tencent.go b/relay/channel/tencent/relay-tencent.go index 0343f5784e7f..0ad5f7ccc311 100644 --- a/relay/channel/tencent/relay-tencent.go +++ b/relay/channel/tencent/relay-tencent.go @@ -145,10 +145,10 @@ func tencentHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Resp return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } if tencentSb.Response.Error.Code != 0 { - return nil, types.WithOpenAIError(types.OpenAIError{ + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(types.OpenAIError{ Message: tencentSb.Response.Error.Message, Code: tencentSb.Response.Error.Code, - }, resp.StatusCode) + }, resp.StatusCode)) } fullTextResponse := responseTencent2OpenAI(&tencentSb.Response) jsonResponse, err := common.Marshal(fullTextResponse) diff --git a/relay/channel/zhipu/relay-zhipu.go b/relay/channel/zhipu/relay-zhipu.go index c3c96a05a90c..9cbcaeaed101 100644 --- a/relay/channel/zhipu/relay-zhipu.go +++ b/relay/channel/zhipu/relay-zhipu.go @@ -231,10 +231,10 @@ func zhipuHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respon return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } if !zhipuResponse.Success { - return nil, types.WithOpenAIError(types.OpenAIError{ + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(types.OpenAIError{ Message: zhipuResponse.Msg, Code: zhipuResponse.Code, - }, resp.StatusCode) + }, resp.StatusCode)) } fullTextResponse := responseZhipu2OpenAI(&zhipuResponse) jsonResponse, err := json.Marshal(fullTextResponse) diff --git a/relay/channel/zhipu_4v/image.go b/relay/channel/zhipu_4v/image.go index b1fd2c8e34b7..be9daebca717 100644 --- a/relay/channel/zhipu_4v/image.go +++ b/relay/channel/zhipu_4v/image.go @@ -67,11 +67,11 @@ func zhipu4vImageHandler(c *gin.Context, resp *http.Response, info *relaycommon. } if zhipuResp.Error != nil && zhipuResp.Error.Message != "" { - return nil, types.WithOpenAIError(types.OpenAIError{ + return nil, types.MarkAsUpstreamError(types.WithOpenAIError(types.OpenAIError{ Message: zhipuResp.Error.Message, Type: "zhipu_image_error", Code: zhipuResp.Error.Code, - }, resp.StatusCode) + }, resp.StatusCode)) } payload := openAIImagePayload{} diff --git a/service/error.go b/service/error.go index cf7325b61039..5c075c363364 100644 --- a/service/error.go +++ b/service/error.go @@ -84,6 +84,9 @@ func ClaudeErrorWrapperLocal(err error, code string, statusCode int) *dto.Claude } func RelayErrorHandler(ctx context.Context, resp *http.Response, showBodyWhenFail bool) (newApiErr *types.NewAPIError) { + defer func() { + types.MarkAsUpstreamError(newApiErr) + }() newApiErr = types.InitOpenAIError(types.ErrorCodeBadResponseStatusCode, resp.StatusCode) responseBody, err := io.ReadAll(resp.Body) diff --git a/service/error_test.go b/service/error_test.go index 9f19bfbb4fec..7020f4dd7f78 100644 --- a/service/error_test.go +++ b/service/error_test.go @@ -122,6 +122,22 @@ func TestRelayErrorHandlerKeepsOpenAIErrorMessage(t *testing.T) { require.Equal(t, message, newAPIError.Error()) } +func TestRelayErrorHandlerMarksStructuredErrorAsUpstreamForDownstreamPolicy(t *testing.T) { + body := `{"error":{"message":"token quota is not enough, url: https://example.invalid/responses","type":"upstream_error","code":"insufficient_quota"}}` + resp := &http.Response{ + StatusCode: http.StatusForbidden, + Body: io.NopCloser(strings.NewReader(body)), + } + + newAPIError := RelayErrorHandler(context.Background(), resp, false) + responseError := types.ApplyDownstreamNewAPIErrorPolicy(newAPIError, "req_upstream") + + require.NotNil(t, newAPIError) + require.Equal(t, http.StatusServiceUnavailable, responseError.StatusCode) + require.Equal(t, "Service temporarily unavailable. Please try again later. (request id: req_upstream)", responseError.Error()) + require.Equal(t, types.ErrorCodeServiceUnavailable, responseError.GetErrorCode()) +} + func TestRelayErrorHandlerKeepsInvalidJSONBodyInDebugLog(t *testing.T) { withDebugEnabled(t, true) diff --git a/service/violation_fee.go b/service/violation_fee.go index 45508856135d..d4810f8c4ca7 100644 --- a/service/violation_fee.go +++ b/service/violation_fee.go @@ -45,7 +45,11 @@ func WrapAsViolationFeeGrokCSAM(err *types.NewAPIError) *types.NewAPIError { oai := err.ToOpenAIError() oai.Type = string(types.ErrorCodeViolationFeeGrokCSAM) oai.Code = string(types.ErrorCodeViolationFeeGrokCSAM) - return types.WithOpenAIError(oai, err.StatusCode, types.ErrOptionWithSkipRetry()) + wrappedErr := types.WithOpenAIError(oai, err.StatusCode, types.ErrOptionWithSkipRetry()) + if types.IsUpstreamError(err) { + types.MarkAsUpstreamError(wrappedErr) + } + return wrappedErr } // NormalizeViolationFeeError ensures: @@ -64,7 +68,11 @@ func NormalizeViolationFeeError(err *types.NewAPIError) *types.NewAPIError { if IsViolationFeeCode(err.GetErrorCode()) { oai := err.ToOpenAIError() - return types.WithOpenAIError(oai, err.StatusCode, types.ErrOptionWithSkipRetry()) + wrappedErr := types.WithOpenAIError(oai, err.StatusCode, types.ErrOptionWithSkipRetry()) + if types.IsUpstreamError(err) { + types.MarkAsUpstreamError(wrappedErr) + } + return wrappedErr } return err diff --git a/types/error.go b/types/error.go index 9717401ae7b2..5b12eea7d033 100644 --- a/types/error.go +++ b/types/error.go @@ -77,6 +77,7 @@ const ( ErrorCodeAwsInvokeError ErrorCode = "aws_invoke_error" ErrorCodeModelNotFound ErrorCode = "model_not_found" ErrorCodePromptBlocked ErrorCode = "prompt_blocked" + ErrorCodeServiceUnavailable ErrorCode = "service_unavailable" // sql error ErrorCodeQueryDataError ErrorCode = "query_data_error" @@ -92,12 +93,54 @@ type NewAPIError struct { RelayError any skipRetry bool recordErrorLog *bool + upstreamError bool errorType ErrorType errorCode ErrorCode StatusCode int Metadata json.RawMessage } +const PublicServiceUnavailableMessage = "Service temporarily unavailable. Please try again later." + +func shouldMarkErrorCodeAsUpstream(errorCode ErrorCode) bool { + switch errorCode { + case ErrorCodeDoRequestFailed, + ErrorCodeBadResponseStatusCode, + ErrorCodeBadResponse, + ErrorCodeReadResponseBodyFailed, + ErrorCodeBadResponseBody: + return true + default: + return false + } +} + +func ApplyDownstreamNewAPIErrorPolicy(err *NewAPIError, requestId string) *NewAPIError { + if err == nil { + return nil + } + if !err.upstreamError { + err.SetMessage(common.MessageWithRequestId(err.Error(), requestId)) + return err + } + return NewErrorWithStatusCode( + errors.New(common.MessageWithRequestId(PublicServiceUnavailableMessage, requestId)), + ErrorCodeServiceUnavailable, + http.StatusServiceUnavailable, + ) +} + +func MarkAsUpstreamError(err *NewAPIError) *NewAPIError { + if err != nil { + err.upstreamError = true + } + return err +} + +func IsUpstreamError(err *NewAPIError) bool { + return err != nil && err.upstreamError +} + // Unwrap enables errors.Is / errors.As to work with NewAPIError by exposing the underlying error. func (e *NewAPIError) Unwrap() error { if e == nil { @@ -245,17 +288,21 @@ func NewError(err error, errorCode ErrorCode, ops ...NewAPIErrorOptions) *NewAPI var newErr *NewAPIError // 保留深层传递的 new err if errors.As(err, &newErr) { + if shouldMarkErrorCodeAsUpstream(errorCode) { + MarkAsUpstreamError(newErr) + } for _, op := range ops { op(newErr) } return newErr } e := &NewAPIError{ - Err: err, - RelayError: nil, - errorType: ErrorTypeNewAPIError, - StatusCode: http.StatusInternalServerError, - errorCode: errorCode, + Err: err, + RelayError: nil, + errorType: ErrorTypeNewAPIError, + StatusCode: http.StatusInternalServerError, + errorCode: errorCode, + upstreamError: shouldMarkErrorCodeAsUpstream(errorCode), } for _, op := range ops { op(e) @@ -267,6 +314,9 @@ func NewOpenAIError(err error, errorCode ErrorCode, statusCode int, ops ...NewAP var newErr *NewAPIError // 保留深层传递的 new err if errors.As(err, &newErr) { + if shouldMarkErrorCodeAsUpstream(errorCode) { + MarkAsUpstreamError(newErr) + } if newErr.RelayError == nil { openaiError := OpenAIError{ Message: newErr.Error(), @@ -303,9 +353,10 @@ func NewErrorWithStatusCode(err error, errorCode ErrorCode, statusCode int, ops Message: err.Error(), Type: string(errorCode), }, - errorType: ErrorTypeNewAPIError, - StatusCode: statusCode, - errorCode: errorCode, + errorType: ErrorTypeNewAPIError, + StatusCode: statusCode, + errorCode: errorCode, + upstreamError: shouldMarkErrorCodeAsUpstream(errorCode), } for _, op := range ops { op(e) @@ -327,11 +378,12 @@ func WithOpenAIError(openAIError OpenAIError, statusCode int, ops ...NewAPIError openAIError.Type = "upstream_error" } e := &NewAPIError{ - RelayError: openAIError, - errorType: ErrorTypeOpenAIError, - StatusCode: statusCode, - Err: errors.New(openAIError.Message), - errorCode: ErrorCode(code), + RelayError: openAIError, + errorType: ErrorTypeOpenAIError, + StatusCode: statusCode, + Err: errors.New(openAIError.Message), + errorCode: ErrorCode(code), + upstreamError: shouldMarkErrorCodeAsUpstream(ErrorCode(code)), } // OpenRouter if len(openAIError.Metadata) > 0 { @@ -351,11 +403,12 @@ func WithClaudeError(claudeError ClaudeError, statusCode int, ops ...NewAPIError claudeError.Type = "upstream_error" } e := &NewAPIError{ - RelayError: claudeError, - errorType: ErrorTypeClaudeError, - StatusCode: statusCode, - Err: errors.New(claudeError.Message), - errorCode: ErrorCode(claudeError.Type), + RelayError: claudeError, + errorType: ErrorTypeClaudeError, + StatusCode: statusCode, + Err: errors.New(claudeError.Message), + errorCode: ErrorCode(claudeError.Type), + upstreamError: shouldMarkErrorCodeAsUpstream(ErrorCode(claudeError.Type)), } for _, op := range ops { op(e) diff --git a/types/error_policy_test.go b/types/error_policy_test.go new file mode 100644 index 000000000000..831f6889076e --- /dev/null +++ b/types/error_policy_test.go @@ -0,0 +1,82 @@ +package types + +import ( + "errors" + "net/http" + "testing" +) + +func TestApplyDownstreamNewAPIErrorPolicyKeepsLocalError(t *testing.T) { + err := NewErrorWithStatusCode( + errors.New("Invalid token"), + ErrorCodeInvalidRequest, + http.StatusUnauthorized, + ) + + got := ApplyDownstreamNewAPIErrorPolicy(err, "req_local") + if got.StatusCode != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", got.StatusCode, http.StatusUnauthorized) + } + if got.Error() != "Invalid token (request id: req_local)" { + t.Fatalf("message = %q", got.Error()) + } + if got.GetErrorCode() != ErrorCodeInvalidRequest { + t.Fatalf("code = %q, want %q", got.GetErrorCode(), ErrorCodeInvalidRequest) + } +} + +func TestApplyDownstreamNewAPIErrorPolicyMapsUpstreamStatusesToServiceUnavailable(t *testing.T) { + tests := []struct { + name string + statusCode int + }{ + {name: "upstream 401", statusCode: http.StatusUnauthorized}, + {name: "upstream 403", statusCode: http.StatusForbidden}, + {name: "upstream 429", statusCode: http.StatusTooManyRequests}, + {name: "upstream 502", statusCode: http.StatusBadGateway}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := NewOpenAIError( + errors.New("unexpected status from upstream"), + ErrorCodeBadResponseStatusCode, + tt.statusCode, + ) + + got := ApplyDownstreamNewAPIErrorPolicy(err, "req_upstream") + if got.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", got.StatusCode, http.StatusServiceUnavailable) + } + if got.Error() != "Service temporarily unavailable. Please try again later. (request id: req_upstream)" { + t.Fatalf("message = %q", got.Error()) + } + if got.GetErrorCode() != ErrorCodeServiceUnavailable { + t.Fatalf("code = %q, want %q", got.GetErrorCode(), ErrorCodeServiceUnavailable) + } + }) + } +} + +func TestApplyDownstreamNewAPIErrorPolicyMapsMarkedProviderError(t *testing.T) { + err := MarkAsUpstreamError(WithOpenAIError(OpenAIError{ + Message: "token quota is not enough, url: https://example.invalid/responses", + Type: "provider_error", + Code: "provider_quota_error", + }, http.StatusOK)) + + got := ApplyDownstreamNewAPIErrorPolicy(err, "req_provider") + if got.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", got.StatusCode, http.StatusServiceUnavailable) + } + openAIError := got.ToOpenAIError() + if openAIError.Message != "Service temporarily unavailable. Please try again later. (request id: req_provider)" { + t.Fatalf("message = %q", openAIError.Message) + } + if openAIError.Type != string(ErrorTypeNewAPIError) { + t.Fatalf("type = %q, want %q", openAIError.Type, ErrorTypeNewAPIError) + } + if openAIError.Code != ErrorCodeServiceUnavailable { + t.Fatalf("code = %v, want %q", openAIError.Code, ErrorCodeServiceUnavailable) + } +}