Mask upstream errors from downstream users

controller/relay.go

@@ -89,18 +89,18 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) {
 	defer func() {
 		if newAPIError != nil {
 			logger.LogError(c, fmt.Sprintf("relay error: %s", common.LocalLogPreview(newAPIError.Error())))
-			newAPIError.SetMessage(common.MessageWithRequestId(newAPIError.Error(), requestId))
+			responseError := types.ApplyDownstreamNewAPIErrorPolicy(newAPIError, requestId)
 			switch relayFormat {
 			case types.RelayFormatOpenAIRealtime:
-				helper.WssError(c, ws, newAPIError.ToOpenAIError())
+				helper.WssError(c, ws, responseError.ToOpenAIError())
 			case types.RelayFormatClaude:
-				c.JSON(newAPIError.StatusCode, gin.H{
+				c.JSON(responseError.StatusCode, gin.H{
 					"type":  "error",
-					"error": newAPIError.ToClaudeError(),
+					"error": responseError.ToClaudeError(),
 				})
 			default:
-				c.JSON(newAPIError.StatusCode, gin.H{
-					"error": newAPIError.ToOpenAIError(),
+				c.JSON(responseError.StatusCode, gin.H{
+					"error": responseError.ToOpenAIError(),
 				})
 			}
 		}
@@ -378,6 +378,15 @@ func processChannelError(c *gin.Context, channelError types.ChannelError, err *t
 		other["error_type"] = err.GetErrorType()
 		other["error_code"] = err.GetErrorCode()
 		other["status_code"] = err.StatusCode
+		if types.IsUpstreamError(err) {
+			other["upstream_error"] = true
+			other["client_status_code"] = http.StatusServiceUnavailable
+			if err.StatusCode != 0 {
+				other["upstream_status_code"] = err.StatusCode
+			}
+		} else {
+			other["upstream_error"] = false
+		}
 		other["channel_id"] = channelId
 		other["channel_name"] = c.GetString("channel_name")
 		other["channel_type"] = c.GetInt("channel_type")

docs/upstream-error-503-logmask.md

@@ -0,0 +1,195 @@
+# Upstream Error 503 Policy
+
+This branch keeps upstream-provider failures private from downstream users.
+
+## Goal
+
+- Local New API errors stay unchanged.
+- Upstream-origin errors returned to downstream clients become HTTP `503` with a generic service-unavailable message.
+- User-visible logs also show upstream-origin errors as HTTP `503` with the same generic message.
+- Admin/server logs and raw database records keep original details for troubleshooting.
+
+## Downstream Behavior
+
+When an upstream error happens, downstream OpenAI-compatible responses should look like:
+
+```json
+{
+  "error": {
+    "message": "Service temporarily unavailable. Please try again later. (request id: ...)",
+    "type": "new_api_error",
+    "code": "service_unavailable"
+  }
+}
+```
+
+Local errors are not masked. For example, an invalid local token still returns:
+
+```json
+{
+  "error": {
+    "message": "Invalid token (request id: ...)",
+    "type": "new_api_error",
+    "code": ""
+  }
+}
+```
+
+## User Log Behavior
+
+For ordinary user log APIs, upstream-origin error logs are masked:
+
+- `content` becomes `status_code=503, Service temporarily unavailable. Please try again later.`
+- `other.status_code` becomes `503`
+- `other.error_code`, `other.error_type`, `other.client_status_code`, and `other.upstream_status_code` are hidden
+- channel fields and `upstream_request_id` are hidden from user-visible results
+
+Admin log APIs keep the original error details, including the real `status_code`; upstream errors also include
+`client_status_code=503` and `upstream_status_code` for troubleshooting.
+
+## Main Files
+
+- `types/error.go`
+  - Adds `upstreamError` marker on `NewAPIError`.
+  - Adds `ApplyDownstreamNewAPIErrorPolicy`.
+  - Adds `MarkAsUpstreamError` and `IsUpstreamError`.
+
+- `controller/relay.go`
+  - Applies the downstream masking policy at the final response boundary.
+  - Records `other.upstream_error`, `other.client_status_code`, and `other.upstream_status_code` for error logs.
+
+- `service/error.go`
+  - Marks errors parsed from upstream non-2xx responses as upstream errors.
+
+- `model/log.go`
+  - Masks upstream-origin error logs only when formatting logs for ordinary users.
+
+- `relay/channel/*`
+  - Marks provider/body-level errors as upstream errors for channels that construct errors directly.
+
+- Tests:
+  - `types/error_policy_test.go`
+  - `model/log_test.go`
+  - `service/error_test.go`
+
+## Local Test Commands
+
+Use the local Go toolchain. If `proxy.golang.org` is slow, use `goproxy.cn`.
+
+```powershell
+$env:GOPROXY='https://goproxy.cn,direct'
+F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./types
+F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./model -run "TestFormatUserLogs"
+F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./service -run "Test(RelayErrorHandler|ResetStatusCode)"
+F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./controller -run "^$"
+```
+
+Known unrelated failures seen in this source snapshot:
+
+- Full `./service` can fail in channel-affinity usage-cache tests.
+- Full `./relay/channel/claude` can fail in existing file-content conversion tests.
+
+For compile-only checks:
+
+```powershell
+$env:GOPROXY='https://goproxy.cn,direct'
+F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe test ./relay/channel/claude -run "^$"
+```
+
+## Frontend Build
+
+The backend embeds both frontend builds. Build them before building the Linux binary:
+
+```powershell
+cd F:\workspace\mszb\.codex-tmp\new-api-src\web\default
+$env:DISABLE_ESLINT_PLUGIN='true'
+$env:VITE_REACT_APP_VERSION=(Get-Content ..\..\VERSION -Raw)
+bun run build
+
+cd F:\workspace\mszb\.codex-tmp\new-api-src\web\classic
+$env:VITE_REACT_APP_VERSION=(Get-Content ..\..\VERSION -Raw)
+bun run build
+```
+
+## Local Linux Build
+
+Do not compile on the server. Build locally:
+
+```powershell
+cd F:\workspace\mszb\.codex-tmp\new-api-src
+$env:GOPROXY='https://goproxy.cn,direct'
+$env:GOOS='linux'
+$env:GOARCH='amd64'
+$env:CGO_ENABLED='0'
+$env:GOEXPERIMENT='greenteagc'
+$version=''; if (Test-Path VERSION) { $raw=Get-Content VERSION -Raw; if ($null -ne $raw) { $version=$raw.Trim() } }
+F:\workspace\mszb\.codex-tmp\go\go\bin\go.exe build -ldflags "-s -w -X 'github.com/QuantumNous/new-api/common.Version=$version'" -o F:\workspace\mszb\.codex-tmp\new-api-linux-amd64
+```
+
+## Deploy To 178
+
+Upload and replace the binary. This does not compile on the server.
+
+```powershell
+scp -i E:\MircDL\178.239.117.128_id_ed25519 -P 80 `
+  F:\workspace\mszb\.codex-tmp\new-api-linux-amd64 `
+  root@178.239.117.128:/opt/new-api/deploy/new-api-upstream-503-logmask
+
+$ts=(Get-Date -Format 'yyyyMMdd-HHmm')
+ssh -i E:\MircDL\178.239.117.128_id_ed25519 -p 80 root@178.239.117.128 "
+docker cp newapi:/new-api /opt/new-api/backup/new-api.bak-$ts &&
+docker cp /opt/new-api/deploy/new-api-upstream-503-logmask newapi:/tmp/new-api-upstream-503-logmask &&
+docker exec newapi sh -lc 'cp /new-api /new-api.bak-$ts && chmod +x /tmp/new-api-upstream-503-logmask && mv /tmp/new-api-upstream-503-logmask /new-api && sha256sum /new-api' &&
+docker restart newapi
+"
+```
+
+Verify:
+
+```powershell
+ssh -i E:\MircDL\178.239.117.128_id_ed25519 -p 80 root@178.239.117.128 `
+  "docker ps --filter name=newapi --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}'"
+```
+
+Invalid local token should still return 401:
+
+```powershell
+ssh -i E:\MircDL\178.239.117.128_id_ed25519 -p 80 root@178.239.117.128 "
+curl -sS -i -X POST http://127.0.0.1:3001/v1/responses \
+  -H 'Authorization: Bearer invalid-token-for-local-check' \
+  -H 'Content-Type: application/json' \
+  -d '{\"model\":\"gpt-5.5\",\"input\":\"hi\"}' | head -n 20
+"
+```
+
+## Rollback
+
+The container keeps timestamped binary backups:
+
+```powershell
+ssh -i E:\MircDL\178.239.117.128_id_ed25519 -p 80 root@178.239.117.128 "
+docker exec newapi sh -lc 'cp /new-api.bak-YYYYMMDD-HHMM /new-api && chmod +x /new-api' &&
+docker restart newapi
+"
+```
+
+Host backups are stored in:
+
+```text
+/opt/new-api/backup/
+```
+
+## Updating From Upstream
+
+Recommended workflow:
+
+```powershell
+cd F:\workspace\mszb\.codex-tmp\new-api-src
+git fetch origin
+git switch codex/upstream-error-503-logmask
+git rebase origin/main
+```
+
+If conflicts happen, resolve them in the files listed in "Main Files", then rerun tests and build/deploy.
+
+Do not deploy the official `calciumion/new-api:latest` directly unless this branch has been rebuilt and redeployed, otherwise these masking changes will be lost.

model/log.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"strings"
 	"time"
 
@@ -72,6 +73,9 @@ func formatUserLogs(logs []*Log, startIdx int) {
 		var otherMap map[string]interface{}
 		otherMap, _ = common.StrToMap(logs[i].Other)
 		if otherMap != nil {
+			if isUserVisibleUpstreamErrorLog(logs[i], otherMap) {
+				maskUserUpstreamErrorLog(logs[i], otherMap)
+			}
 			// Remove admin-only debug fields.
 			delete(otherMap, "admin_info")
 			// delete(otherMap, "reject_reason")
@@ -82,6 +86,48 @@ func formatUserLogs(logs []*Log, startIdx int) {
 	}
 }
 
+func isUserVisibleUpstreamErrorLog(log *Log, otherMap map[string]interface{}) bool {
+	if log == nil || log.Type != LogTypeError || otherMap == nil {
+		return false
+	}
+	if upstream, ok := otherMap["upstream_error"]; ok {
+		switch v := upstream.(type) {
+		case bool:
+			return v
+		case string:
+			return strings.EqualFold(v, "true")
+		}
+	}
+	code, _ := otherMap["error_code"].(string)
+	switch types.ErrorCode(code) {
+	case types.ErrorCodeDoRequestFailed,
+		types.ErrorCodeBadResponseStatusCode,
+		types.ErrorCodeBadResponse,
+		types.ErrorCodeReadResponseBodyFailed,
+		types.ErrorCodeBadResponseBody:
+		return true
+	default:
+		return false
+	}
+}
+
+func maskUserUpstreamErrorLog(log *Log, otherMap map[string]interface{}) {
+	log.Content = fmt.Sprintf("status_code=%d, %s", http.StatusServiceUnavailable, types.PublicServiceUnavailableMessage)
+	log.ChannelId = 0
+	log.ChannelName = ""
+	log.UpstreamRequestId = ""
+
+	otherMap["status_code"] = http.StatusServiceUnavailable
+	delete(otherMap, "client_status_code")
+	delete(otherMap, "upstream_status_code")
+	delete(otherMap, "error_code")
+	delete(otherMap, "error_type")
+	delete(otherMap, "channel_id")
+	delete(otherMap, "channel_name")
+	delete(otherMap, "channel_type")
+	delete(otherMap, "upstream_error")
+}
+
 func GetLogByTokenId(tokenId int) (logs []*Log, err error) {
 	err = LOG_DB.Model(&Log{}).Where("token_id = ?", tokenId).Order("id desc").Limit(common.MaxRecentItems).Find(&logs).Error
 	formatUserLogs(logs, 0)