feat: business payload logging toggle (privacy-first, default off)

common/constants.go

@@ -92,6 +92,10 @@ var MemoryCacheEnabled bool
 
 var LogConsumeEnabled = true
 
+// PayloadLogEnabled is the platform-wide business-payload logging switch.
+// OFF by default: relay request/response bodies are never captured or stored.
+var PayloadLogEnabled = false
+
 var TLSInsecureSkipVerify bool
 var InsecureTLSConfig = &tls.Config{InsecureSkipVerify: true}
 

controller/payload_log.go

@@ -0,0 +1,112 @@
+package controller
+
+import (
+	"strconv"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/QuantumNous/new-api/model"
+
+	"github.com/gin-gonic/gin"
+)
+
+// GetPayloadLogs returns a paginated, body-free list of ALL captured payload
+// logs. Admin-only.
+func GetPayloadLogs(c *gin.Context) {
+	pageInfo := common.GetPageQuery(c)
+	username := c.Query("username")
+	modelName := c.Query("model_name")
+	requestId := c.Query("request_id")
+	startTimestamp, _ := strconv.ParseInt(c.Query("start_timestamp"), 10, 64)
+	endTimestamp, _ := strconv.ParseInt(c.Query("end_timestamp"), 10, 64)
+	logs, total, err := model.GetPayloadLogs(0, username, modelName, requestId, startTimestamp, endTimestamp, pageInfo.GetStartIdx(), pageInfo.GetPageSize())
+	if err != nil {
+		common.ApiError(c, err)
+		return
+	}
+	pageInfo.SetTotal(int(total))
+	pageInfo.SetItems(logs)
+	common.ApiSuccess(c, pageInfo)
+}
+
+// GetSelfPayloadLogs returns the caller's OWN payload logs only. Any user.
+func GetSelfPayloadLogs(c *gin.Context) {
+	pageInfo := common.GetPageQuery(c)
+	userId := c.GetInt("id")
+	modelName := c.Query("model_name")
+	requestId := c.Query("request_id")
+	startTimestamp, _ := strconv.ParseInt(c.Query("start_timestamp"), 10, 64)
+	endTimestamp, _ := strconv.ParseInt(c.Query("end_timestamp"), 10, 64)
+	logs, total, err := model.GetPayloadLogs(userId, "", modelName, requestId, startTimestamp, endTimestamp, pageInfo.GetStartIdx(), pageInfo.GetPageSize())
+	if err != nil {
+		common.ApiError(c, err)
+		return
+	}
+	pageInfo.SetTotal(int(total))
+	pageInfo.SetItems(logs)
+	common.ApiSuccess(c, pageInfo)
+}
+
+// GetPayloadLogDetail returns a single log with full bodies. Admin-only.
+func GetPayloadLogDetail(c *gin.Context) {
+	id, _ := strconv.Atoi(c.Param("id"))
+	log, err := model.GetPayloadLogById(id, 0)
+	if err != nil {
+		common.ApiError(c, err)
+		return
+	}
+	common.ApiSuccess(c, log)
+}
+
+// GetSelfPayloadLogDetail returns a single log with full bodies, but only if it
+// belongs to the caller. Any user.
+func GetSelfPayloadLogDetail(c *gin.Context) {
+	id, _ := strconv.Atoi(c.Param("id"))
+	log, err := model.GetPayloadLogById(id, c.GetInt("id"))
+	if err != nil {
+		common.ApiError(c, err)
+		return
+	}
+	common.ApiSuccess(c, log)
+}
+
+// GetPayloadLogSwitchStatus reports the current platform-wide switch state.
+// Readable by any authenticated user (transparency).
+func GetPayloadLogSwitchStatus(c *gin.Context) {
+	common.ApiSuccess(c, gin.H{"enabled": common.PayloadLogEnabled})
+}
+
+// SetPayloadLogSwitch flips the platform-wide switch and records who did it.
+// Root only.
+func SetPayloadLogSwitch(c *gin.Context) {
+	var req struct {
+		Enabled bool `json:"enabled"`
+	}
+	if err := common.DecodeJson(c.Request.Body, &req); err != nil {
+		common.ApiErrorMsg(c, "invalid parameter")
+		return
+	}
+	value := "false"
+	if req.Enabled {
+		value = "true"
+	}
+	if err := model.UpdateOption("PayloadLogEnabled", value); err != nil {
+		common.ApiError(c, err)
+		return
+	}
+	model.RecordPayloadLogSwitchAudit(c.GetInt("id"), c.GetString("username"), req.Enabled)
+	common.ApiSuccess(c, gin.H{"enabled": req.Enabled})
+}
+
+// GetPayloadLogSwitchAudits returns the switch change history (who turned it on
+// or off, and when). Readable by any authenticated user (transparency).
+func GetPayloadLogSwitchAudits(c *gin.Context) {
+	pageInfo := common.GetPageQuery(c)
+	audits, total, err := model.GetPayloadLogSwitchAudits(pageInfo.GetStartIdx(), pageInfo.GetPageSize())
+	if err != nil {
+		common.ApiError(c, err)
+		return
+	}
+	pageInfo.SetTotal(int(total))
+	pageInfo.SetItems(audits)
+	common.ApiSuccess(c, pageInfo)
+}

middleware/payload_log.go

@@ -0,0 +1,80 @@
+package middleware
+
+import (
+	"bytes"
+	"time"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/QuantumNous/new-api/model"
+
+	"github.com/bytedance/gopkg/util/gopool"
+	"github.com/gin-gonic/gin"
+)
+
+// payloadLogMaxBodySize bounds how much of each request/response body is stored,
+// so a huge upload or long stream cannot bloat the database.
+const payloadLogMaxBodySize = 256 * 1024 // 256KB per body
+
+// PayloadLog captures the full request and response bodies of relay calls when
+// the platform-wide "business payload logging" switch (common.PayloadLogEnabled)
+// is ON. When it is OFF (the default) this middleware is a strict no-op with
+// zero overhead: no body is read, no writer is wrapped, nothing is stored — so
+// user prompts and model responses are never persisted.
+func PayloadLog() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// Default-OFF fast path: zero overhead, no capture.
+		if !common.PayloadLogEnabled || c.Request.Method != "POST" {
+			c.Next()
+			return
+		}
+
+		start := time.Now()
+
+		// Read the request body before c.Next(); GetBodyStorage caches it, so
+		// the relay downstream still reads the same body. (BodyStorageCleanup
+		// releases the storage only after the request finishes.)
+		var requestBody string
+		if bs, err := common.GetBodyStorage(c); err == nil {
+			if b, err := bs.Bytes(); err == nil {
+				requestBody = truncatePayloadBody(b)
+			}
+		}
+
+		// Tee a bounded copy of the response. auditResponseWriter (audit.go)
+		// already implements exactly this capped-buffer wrapper.
+		writer := &auditResponseWriter{
+			ResponseWriter: c.Writer,
+			body:           bytes.NewBuffer(nil),
+			maxSize:        payloadLogMaxBodySize,
+		}
+		c.Writer = writer
+
+		c.Next()
+
+		entry := &model.PayloadLog{
+			CreatedAt:    common.GetTimestamp(),
+			UserId:       c.GetInt("id"),
+			Username:     c.GetString("username"),
+			TokenName:    c.GetString("token_name"),
+			ModelName:    c.GetString("original_model"),
+			ChannelId:    c.GetInt("channel_id"),
+			RequestId:    c.GetString(common.RequestIdKey),
+			Ip:           c.ClientIP(),
+			StatusCode:   writer.Status(),
+			DurationMs:   time.Since(start).Milliseconds(),
+			RequestBody:  requestBody,
+			ResponseBody: writer.body.String(),
+		}
+		// Persist off the request path so logging never adds latency.
+		gopool.Go(func() {
+			model.RecordPayloadLog(entry)
+		})
+	}
+}
+
+func truncatePayloadBody(b []byte) string {
+	if len(b) > payloadLogMaxBodySize {
+		return string(b[:payloadLogMaxBodySize])
+	}
+	return string(b)
+}

model/main.go

@@ -270,6 +270,8 @@ func migrateDB() error {
 		&Redemption{},
 		&Ability{},
 		&Log{},
+		&PayloadLog{},
+		&PayloadLogSwitchAudit{},
 		&Midjourney{},
 		&TopUp{},
 		&QuotaData{},
@@ -400,7 +402,7 @@ func migrateLOGDB() error {
 	if common.UsingLogDatabase(common.DatabaseTypeClickHouse) {
 		return migrateClickHouseLogDB()
 	}
-	return LOG_DB.AutoMigrate(&Log{})
+	return LOG_DB.AutoMigrate(&Log{}, &PayloadLog{}, &PayloadLogSwitchAudit{})
 }
 
 func migrateClickHouseLogDB() error {

model/option.go

@@ -48,6 +48,7 @@ func InitOptionMap() {
 	common.OptionMap["AutomaticDisableChannelEnabled"] = strconv.FormatBool(common.AutomaticDisableChannelEnabled)
 	common.OptionMap["AutomaticEnableChannelEnabled"] = strconv.FormatBool(common.AutomaticEnableChannelEnabled)
 	common.OptionMap["LogConsumeEnabled"] = strconv.FormatBool(common.LogConsumeEnabled)
+	common.OptionMap["PayloadLogEnabled"] = strconv.FormatBool(common.PayloadLogEnabled)
 	common.OptionMap["DisplayInCurrencyEnabled"] = strconv.FormatBool(common.DisplayInCurrencyEnabled)
 	common.OptionMap["DisplayTokenStatEnabled"] = strconv.FormatBool(common.DisplayTokenStatEnabled)
 	common.OptionMap["DrawingEnabled"] = strconv.FormatBool(common.DrawingEnabled)
@@ -336,6 +337,8 @@ func updateOptionMap(key string, value string) (err error) {
 			common.AutomaticEnableChannelEnabled = boolValue
 		case "LogConsumeEnabled":
 			common.LogConsumeEnabled = boolValue
+		case "PayloadLogEnabled":
+			common.PayloadLogEnabled = boolValue
 		case "DisplayInCurrencyEnabled":
 			// 兼容旧字段：同步到新配置 general_setting.quota_display_type（运行时生效）
 			// true -> USD, false -> TOKENS

model/payload_log.go

@@ -0,0 +1,131 @@
+package model
+
+import (
+	"github.com/QuantumNous/new-api/common"
+
+	"gorm.io/gorm"
+)
+
+// PayloadLog stores the full request and response bodies of a relay call. Rows
+// are only ever written when the platform-wide switch common.PayloadLogEnabled
+// is ON; with the switch OFF (default) no payload is captured or persisted, so
+// the platform keeps only billing/ops metadata (the Log table).
+type PayloadLog struct {
+	Id           int    `json:"id"`
+	CreatedAt    int64  `json:"created_at" gorm:"bigint;index"`
+	UserId       int    `json:"user_id" gorm:"index"`
+	Username     string `json:"username" gorm:"index;default:''"`
+	TokenName    string `json:"token_name" gorm:"default:''"`
+	ModelName    string `json:"model_name" gorm:"index;default:''"`
+	ChannelId    int    `json:"channel_id" gorm:"index;default:0"`
+	RequestId    string `json:"request_id" gorm:"type:varchar(64);index;default:''"`
+	Ip           string `json:"ip" gorm:"default:''"`
+	StatusCode   int    `json:"status_code" gorm:"default:0"`
+	DurationMs   int64  `json:"duration_ms" gorm:"default:0"`
+	RequestBody  string `json:"request_body,omitempty" gorm:"type:text"`
+	ResponseBody string `json:"response_body,omitempty" gorm:"type:text"`
+}
+
+func (PayloadLog) TableName() string {
+	return "payload_logs"
+}
+
+// PayloadLogSwitchAudit records every change of the PayloadLogEnabled switch:
+// who flipped it, to what state, and when. It is readable by any authenticated
+// user so customers can independently verify the platform's logging behaviour.
+type PayloadLogSwitchAudit struct {
+	Id        int    `json:"id"`
+	CreatedAt int64  `json:"created_at" gorm:"bigint;index"`
+	UserId    int    `json:"user_id" gorm:"index"`
+	Username  string `json:"username" gorm:"index;default:''"`
+	Enabled   bool   `json:"enabled"`
+}
+
+func (PayloadLogSwitchAudit) TableName() string {
+	return "payload_log_switch_audits"
+}
+
+// payloadLogListColumns excludes the two body columns so the list view stays
+// light; full bodies are only loaded on demand via GetPayloadLogById.
+const payloadLogListColumns = "id, created_at, user_id, username, token_name, model_name, channel_id, request_id, ip, status_code, duration_ms"
+
+// RecordPayloadLog persists a captured payload. Errors are swallowed with a log
+// line: payload logging must never affect the live relay request.
+func RecordPayloadLog(log *PayloadLog) {
+	if log == nil {
+		return
+	}
+	if err := LOG_DB.Create(log).Error; err != nil {
+		common.SysLog("failed to record payload log: " + err.Error())
+	}
+}
+
+// GetPayloadLogs returns a page of payload logs WITHOUT the request/response
+// bodies. A non-zero userId scopes the result to that user (self view); pass 0
+// for the admin all-users view.
+func GetPayloadLogs(userId int, username, modelName, requestId string, startTimestamp, endTimestamp int64, startIdx, pageSize int) (logs []*PayloadLog, total int64, err error) {
+	tx := LOG_DB.Model(&PayloadLog{})
+	if userId != 0 {
+		tx = tx.Where("user_id = ?", userId)
+	}
+	if username != "" {
+		tx = tx.Where("username = ?", username)
+	}
+	if modelName != "" {
+		tx = tx.Where("model_name = ?", modelName)
+	}
+	if requestId != "" {
+		tx = tx.Where("request_id = ?", requestId)
+	}
+	if startTimestamp != 0 {
+		tx = tx.Where("created_at >= ?", startTimestamp)
+	}
+	if endTimestamp != 0 {
+		tx = tx.Where("created_at <= ?", endTimestamp)
+	}
+	if err = tx.Count(&total).Error; err != nil {
+		return nil, 0, err
+	}
+	err = tx.Select(payloadLogListColumns).Order("id desc").Limit(pageSize).Offset(startIdx).Find(&logs).Error
+	return logs, total, err
+}
+
+// GetPayloadLogById returns a single row with full bodies. A non-zero userId
+// enforces ownership (self view); pass 0 to allow any row (admin view).
+func GetPayloadLogById(id int, userId int) (*PayloadLog, error) {
+	if id == 0 {
+		return nil, gorm.ErrRecordNotFound
+	}
+	tx := LOG_DB.Where("id = ?", id)
+	if userId != 0 {
+		tx = tx.Where("user_id = ?", userId)
+	}
+	var log PayloadLog
+	if err := tx.First(&log).Error; err != nil {
+		return nil, err
+	}
+	return &log, nil
+}
+
+// RecordPayloadLogSwitchAudit appends an entry to the switch change history.
+func RecordPayloadLogSwitchAudit(userId int, username string, enabled bool) {
+	audit := &PayloadLogSwitchAudit{
+		CreatedAt: common.GetTimestamp(),
+		UserId:    userId,
+		Username:  username,
+		Enabled:   enabled,
+	}
+	if err := LOG_DB.Create(audit).Error; err != nil {
+		common.SysLog("failed to record payload log switch audit: " + err.Error())
+	}
+}
+
+// GetPayloadLogSwitchAudits returns the paginated switch change history.
+func GetPayloadLogSwitchAudits(startIdx, pageSize int) (audits []*PayloadLogSwitchAudit, total int64, err error) {
+	tx := LOG_DB.Model(&PayloadLogSwitchAudit{})
+	if err = tx.Count(&total).Error; err != nil {
+		return nil, 0, err
+	}
+	err = tx.Order("id desc").Limit(pageSize).Offset(startIdx).Find(&audits).Error
+	return audits, total, err
+}

router/api-router.go

@@ -278,6 +278,15 @@ func SetApiRouter(router *gin.Engine) {
 		logRoute.GET("/self", middleware.UserAuth(), controller.GetUserLogs)
 		logRoute.GET("/self/search", middleware.UserAuth(), middleware.SearchRateLimit(), controller.SearchUserLogs)
 
+		payloadLogRoute := apiRouter.Group("/payload_log")
+		payloadLogRoute.GET("/", middleware.AdminAuth(), controller.GetPayloadLogs)
+		payloadLogRoute.GET("/detail/:id", middleware.AdminAuth(), controller.GetPayloadLogDetail)
+		payloadLogRoute.GET("/self", middleware.UserAuth(), controller.GetSelfPayloadLogs)
+		payloadLogRoute.GET("/self/detail/:id", middleware.UserAuth(), controller.GetSelfPayloadLogDetail)
+		payloadLogRoute.GET("/switch", middleware.UserAuth(), controller.GetPayloadLogSwitchStatus)
+		payloadLogRoute.POST("/switch", middleware.RootAuth(), controller.SetPayloadLogSwitch)
+		payloadLogRoute.GET("/switch/audits", middleware.UserAuth(), controller.GetPayloadLogSwitchAudits)
+
 		systemTaskRoute := apiRouter.Group("/system-task")
 		systemTaskRoute.Use(middleware.RootAuth())
 		{

router/relay-router.go

@@ -83,6 +83,7 @@ func SetRelayRouter(router *gin.Engine) {
 		//http router
 		httpRouter := relayV1Router.Group("")
 		httpRouter.Use(middleware.Distribute())
+		httpRouter.Use(middleware.PayloadLog())
 
 		// claude related routes
 		httpRouter.POST("/messages", func(c *gin.Context) {