diff --git a/common/endpoint_defaults.go b/common/endpoint_defaults.go index 11ec79217530..3e63f3aaf0d9 100644 --- a/common/endpoint_defaults.go +++ b/common/endpoint_defaults.go @@ -25,6 +25,7 @@ var defaultEndpointInfoMap = map[constant.EndpointType]EndpointInfo{ constant.EndpointTypeJinaRerank: {Path: "/v1/rerank", Method: "POST"}, constant.EndpointTypeImageGeneration: {Path: "/v1/images/generations", Method: "POST"}, constant.EndpointTypeEmbeddings: {Path: "/v1/embeddings", Method: "POST"}, + constant.EndpointTypeOpenAIVideo: {Path: "/v1/videos", Method: "POST"}, } // GetDefaultEndpointInfo 返回指定端点类型的默认信息以及是否存在 diff --git a/controller/channel-test.go b/controller/channel-test.go index 8d62a4acf60d..82db04db0cf2 100644 --- a/controller/channel-test.go +++ b/controller/channel-test.go @@ -42,6 +42,18 @@ type testResult struct { newAPIError *types.NewAPIError } +func isImageOrVideoGenerationModel(model string) bool { + lower := strings.ToLower(model) + return strings.Contains(model, "gpt-image") || + strings.Contains(model, "dall-e") || + strings.Contains(lower, "stable-diffusion") || + strings.Contains(lower, "flux") || + strings.HasPrefix(lower, "veo-") || + strings.HasPrefix(lower, "sora-") || + strings.Contains(lower, "imagen") || + strings.Contains(lower, "seedream") +} + func normalizeChannelTestEndpoint(channel *model.Channel, modelName, endpointType string) string { normalized := strings.TrimSpace(endpointType) if normalized != "" { @@ -116,8 +128,8 @@ func testChannel(channel *model.Channel, testModel string, endpointType string, requestPath = "/v1/embeddings" // 修改请求路径 } - // VolcEngine 图像生成模型 - if channel.Type == constant.ChannelTypeVolcEngine && strings.Contains(testModel, "seedream") { + // 通用图像/视频生成模型检测 + if isImageOrVideoGenerationModel(testModel) { requestPath = "/v1/images/generations" } @@ -685,6 +697,16 @@ func buildTestRequest(model string, endpointType string, channel *model.Channel, } } + // 图像/视频生成模型 + if isImageOrVideoGenerationModel(model) { + return &dto.ImageRequest{ + Model: model, + Prompt: "a cute cat", + N: lo.ToPtr(uint(1)), + Size: "1024x1024", + } + } + // Responses compaction models (must use /v1/responses/compact) if strings.HasSuffix(model, ratio_setting.CompactModelSuffix) { return &dto.OpenAIResponsesCompactionRequest{ diff --git a/controller/playground.go b/controller/playground.go index 501c4e156573..4c183145fe33 100644 --- a/controller/playground.go +++ b/controller/playground.go @@ -3,9 +3,14 @@ package controller import ( "errors" "fmt" + "io" + "net/http" + "strconv" + "strings" "github.com/QuantumNous/new-api/middleware" "github.com/QuantumNous/new-api/model" + "github.com/QuantumNous/new-api/relay" relaycommon "github.com/QuantumNous/new-api/relay/common" "github.com/QuantumNous/new-api/types" @@ -54,3 +59,101 @@ func Playground(c *gin.Context) { Relay(c, types.RelayFormatOpenAI) } + +// PlaygroundVideoProxy proxies an authenticated video content request to the upstream. +// Route: GET /pg/video/:channelId/:videoId/content +// The backend fetches the video binary with the channel's Bearer token and streams it back. +func PlaygroundVideoProxy(c *gin.Context) { + channelIdStr := c.Param("channelId") + videoId := c.Param("videoId") + + if channelIdStr == "" || videoId == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "missing channelId or videoId"}) + return + } + + channelId, err := strconv.Atoi(channelIdStr) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid channelId"}) + return + } + + channel, err := model.GetChannelById(channelId, true) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "channel not found"}) + return + } + + baseURL := "" + if channel.BaseURL != nil { + baseURL = strings.TrimRight(*channel.BaseURL, "/") + } + if baseURL == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "channel has no base URL"}) + return + } + + keys := channel.GetKeys() + if len(keys) == 0 { + c.JSON(http.StatusInternalServerError, gin.H{"error": "channel has no API key"}) + return + } + apiKey := keys[0] + + upstreamURL := fmt.Sprintf("%s/videos/%s/content", baseURL, videoId) + + req, err := http.NewRequestWithContext(c.Request.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create upstream request"}) + return + } + req.Header.Set("Authorization", "Bearer "+apiKey) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": "upstream request failed: " + err.Error()}) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + c.Data(resp.StatusCode, resp.Header.Get("Content-Type"), body) + return + } + + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + contentType = "video/mp4" + } + c.Header("Content-Type", contentType) + if cl := resp.Header.Get("Content-Length"); cl != "" { + c.Header("Content-Length", cl) + } + c.Header("Cache-Control", "no-store") + + c.Status(http.StatusOK) + io.Copy(c.Writer, resp.Body) //nolint:errcheck +} + +// PlaygroundAudioProxy serves a cached TTS audio file by its UUID. +// Route: GET /pg/audio/:audioId +func PlaygroundAudioProxy(c *gin.Context) { + audioId := c.Param("audioId") + + // Strip any file extension from the ID (e.g. "uuid.mp3" → "uuid") + if idx := strings.LastIndex(audioId, "."); idx != -1 { + audioId = audioId[:idx] + } + + data, contentType, ok := relay.GetCachedAudio(audioId) + if !ok { + c.JSON(http.StatusNotFound, gin.H{"error": "audio not found or expired"}) + return + } + + c.Header("Cache-Control", "no-store") + c.Data(http.StatusOK, contentType, data) +} + diff --git a/middleware/request_logger.go b/middleware/request_logger.go new file mode 100644 index 000000000000..13c2dd02f01d --- /dev/null +++ b/middleware/request_logger.go @@ -0,0 +1,89 @@ +package middleware + +import ( + "bytes" + "fmt" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/logger" + "github.com/gin-gonic/gin" +) + +// responseBodyWriter wraps gin.ResponseWriter to capture the response body. +type responseBodyWriter struct { + gin.ResponseWriter + buf *bytes.Buffer +} + +func (w *responseBodyWriter) Write(b []byte) (int, error) { + w.buf.Write(b) + return w.ResponseWriter.Write(b) +} + +func (w *responseBodyWriter) WriteString(s string) (int, error) { + w.buf.WriteString(s) + return w.ResponseWriter.WriteString(s) +} + +// RequestLogger logs the inbound request (URL, headers, body) and the +// outbound response body when DEBUG=true. +func RequestLogger() gin.HandlerFunc { + return func(c *gin.Context) { + if !common.DebugEnabled { + c.Next() + return + } + + // --- request --- + var sb strings.Builder + sb.WriteString("\n========== Inbound Request ==========\n") + sb.WriteString(fmt.Sprintf("%s %s\n", c.Request.Method, c.Request.URL.String())) + + sb.WriteString("--- Headers ---\n") + for key, values := range c.Request.Header { + for _, v := range values { + if strings.EqualFold(key, "Authorization") || strings.EqualFold(key, "x-api-key") { + if len(v) > 12 { + v = v[:12] + "***" + } + } + sb.WriteString(fmt.Sprintf("%s: %s\n", key, v)) + } + } + + sb.WriteString("--- Body ---\n") + bodyStorage, err := common.GetBodyStorage(c) + if err == nil { + bodyBytes, err := bodyStorage.Bytes() + if err == nil { + sb.Write(bodyBytes) + sb.WriteString("\n") + } + } + sb.WriteString("=====================================") + logger.LogDebug(c.Request.Context(), sb.String()) + + // --- wrap response writer to capture output --- + rbw := &responseBodyWriter{ResponseWriter: c.Writer, buf: &bytes.Buffer{}} + c.Writer = rbw + + c.Next() + + // --- response --- + var sb2 strings.Builder + sb2.WriteString("\n========== Inbound Response ==========\n") + sb2.WriteString(fmt.Sprintf("Status: %d\n", rbw.Status())) + sb2.WriteString("--- Body ---\n") + const maxRespBytes = 4 << 10 // 4 KB + respBytes := rbw.buf.Bytes() + if len(respBytes) > maxRespBytes { + sb2.Write(respBytes[:maxRespBytes]) + sb2.WriteString(fmt.Sprintf("\n... (truncated, total %d bytes)", len(respBytes))) + } else { + sb2.Write(respBytes) + } + sb2.WriteString("\n======================================") + logger.LogDebug(c.Request.Context(), sb2.String()) + } +} diff --git a/model/main.go b/model/main.go index f37cb667cd43..3e9445959551 100644 --- a/model/main.go +++ b/model/main.go @@ -202,7 +202,7 @@ func InitDB() (err error) { //_, _ = sqlDB.Exec("ALTER TABLE channels MODIFY model_mapping TEXT;") // TODO: delete this line when most users have upgraded } common.SysLog("database migration started") - err = migrateDB() + // err = migrateDB() return err } else { common.FatalLog(err) @@ -239,7 +239,7 @@ func InitLogDB() (err error) { return nil } common.SysLog("database migration started") - err = migrateLOGDB() + // err = migrateLOGDB() return err } else { common.FatalLog(err) diff --git a/model/option.go b/model/option.go index 37fb6cf5bdc6..077cc4445e7a 100644 --- a/model/option.go +++ b/model/option.go @@ -152,6 +152,8 @@ func InitOptionMap() { common.OptionMap["ImageRatio"] = ratio_setting.ImageRatio2JSONString() common.OptionMap["AudioRatio"] = ratio_setting.AudioRatio2JSONString() common.OptionMap["AudioCompletionRatio"] = ratio_setting.AudioCompletionRatio2JSONString() + common.OptionMap["ContextTierRatio"] = ratio_setting.ContextTierRatio2JSONString() + common.OptionMap["AudioMinutePrice"] = ratio_setting.AudioMinutePrice2JSONString() common.OptionMap["TopUpLink"] = common.TopUpLink //common.OptionMap["ChatLink"] = common.ChatLink //common.OptionMap["ChatLink2"] = common.ChatLink2 @@ -521,6 +523,10 @@ func updateOptionMap(key string, value string) (err error) { err = ratio_setting.UpdateAudioRatioByJSONString(value) case "AudioCompletionRatio": err = ratio_setting.UpdateAudioCompletionRatioByJSONString(value) + case "ContextTierRatio": + err = ratio_setting.UpdateContextTierRatioByJSONString(value) + case "AudioMinutePrice": + err = ratio_setting.UpdateAudioMinutePriceByJSONString(value) case "TopUpLink": common.TopUpLink = value //case "ChatLink": diff --git a/relay/channel/api_request.go b/relay/channel/api_request.go index 8dfb61d40093..b45d0526f8a4 100644 --- a/relay/channel/api_request.go +++ b/relay/channel/api_request.go @@ -292,10 +292,25 @@ func DoApiRequest(a Adaptor, c *gin.Context, info *common.RelayInfo, requestBody if err != nil { return nil, fmt.Errorf("get request url failed: %w", err) } + + var bodyBytes []byte + if requestBody != nil { + bodyBytes, err = io.ReadAll(requestBody) + if err != nil { + return nil, fmt.Errorf("read request body failed: %w", err) + } + } + if common2.DebugEnabled { - println("fullRequestURL:", fullRequestURL) + var sb strings.Builder + sb.WriteString("\n========== Upstream Request ==========\n") + sb.WriteString(fmt.Sprintf("URL: %s %s\n", c.Request.Method, fullRequestURL)) + sb.WriteString(fmt.Sprintf("Body: %s\n", string(bodyBytes))) + sb.WriteString("======================================") + logger.LogDebug(c.Request.Context(), sb.String()) } - req, err := http.NewRequest(c.Request.Method, fullRequestURL, requestBody) + + req, err := http.NewRequest(c.Request.Method, fullRequestURL, strings.NewReader(string(bodyBytes))) if err != nil { return nil, fmt.Errorf("new request failed: %w", err) } @@ -311,6 +326,25 @@ func DoApiRequest(a Adaptor, c *gin.Context, info *common.RelayInfo, requestBody return nil, err } applyHeaderOverrideToRequest(req, headerOverride) + + if common2.DebugEnabled { + var sb strings.Builder + sb.WriteString("\n========== Upstream Request Headers ==========\n") + for key, values := range req.Header { + for _, v := range values { + if strings.EqualFold(key, "Authorization") { + // mask the key, only show prefix + if len(v) > 12 { + v = v[:12] + "***" + } + } + sb.WriteString(fmt.Sprintf("%s: %s\n", key, v)) + } + } + sb.WriteString("==============================================") + logger.LogDebug(c.Request.Context(), sb.String()) + } + resp, err := doRequest(c, req, info) if err != nil { return nil, fmt.Errorf("do request failed: %w", err) diff --git a/relay/channel/openai/audio.go b/relay/channel/openai/audio.go index 3bab3c1a5253..7ea61bf4edc1 100644 --- a/relay/channel/openai/audio.go +++ b/relay/channel/openai/audio.go @@ -121,19 +121,42 @@ func OpenaiSTTHandler(c *gin.Context, resp *http.Response, info *relaycommon.Rel // 写入新的 response body service.IOCopyBytesGracefully(c, resp, responseBody) - var responseData struct { - Usage *dto.Usage `json:"usage"` - } - if err := common.Unmarshal(responseBody, &responseData); err == nil && responseData.Usage != nil { - if responseData.Usage.TotalTokens > 0 { - usage := responseData.Usage - if usage.PromptTokens == 0 { - usage.PromptTokens = usage.InputTokens + // 尝试从 verbose_json 响应中提取音频时长，用于按分钟计费 + if responseFormat == "verbose_json" || responseFormat == "" { + var verboseResp struct { + Duration float64 `json:"duration"` + Usage *dto.Usage `json:"usage"` + } + if err := common.Unmarshal(responseBody, &verboseResp); err == nil { + if verboseResp.Duration > 0 { + info.AudioDurationSeconds = verboseResp.Duration + } + if verboseResp.Usage != nil && verboseResp.Usage.TotalTokens > 0 { + usage := verboseResp.Usage + if usage.PromptTokens == 0 { + usage.PromptTokens = usage.InputTokens + } + if usage.CompletionTokens == 0 { + usage.CompletionTokens = usage.OutputTokens + } + return nil, usage } - if usage.CompletionTokens == 0 { - usage.CompletionTokens = usage.OutputTokens + } + } else { + var responseData struct { + Usage *dto.Usage `json:"usage"` + } + if err := common.Unmarshal(responseBody, &responseData); err == nil && responseData.Usage != nil { + if responseData.Usage.TotalTokens > 0 { + usage := responseData.Usage + if usage.PromptTokens == 0 { + usage.PromptTokens = usage.InputTokens + } + if usage.CompletionTokens == 0 { + usage.CompletionTokens = usage.OutputTokens + } + return nil, usage } - return nil, usage } } diff --git a/relay/channel/openai/relay-openai.go b/relay/channel/openai/relay-openai.go index d33c5555f267..d3ce7c1518b5 100644 --- a/relay/channel/openai/relay-openai.go +++ b/relay/channel/openai/relay-openai.go @@ -1,10 +1,12 @@ package openai import ( + "context" "fmt" "io" "net/http" "strings" + "time" "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/constant" @@ -12,6 +14,7 @@ import ( "github.com/QuantumNous/new-api/logger" "github.com/QuantumNous/new-api/relay/channel/openrouter" relaycommon "github.com/QuantumNous/new-api/relay/common" + relayconstant "github.com/QuantumNous/new-api/relay/constant" "github.com/QuantumNous/new-api/relay/helper" "github.com/QuantumNous/new-api/service" @@ -103,6 +106,174 @@ func sendStreamData(c *gin.Context, info *relaycommon.RelayInfo, data string, fo return helper.ObjectData(c, lastStreamResponse) } +// videoStatusResponse is the shape of a GET /videos/{id} response. +type videoStatusResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} + +// pollVideoUntilComplete polls GET {baseURL}/videos/{id} with Bearer auth until +// the video status is "completed" or "failed". It sends SSE progress chunks +// via helper.ObjectData while polling, then returns the content URL on success. +func pollVideoUntilComplete(c *gin.Context, info *relaycommon.RelayInfo, videoID string) (string, error) { + baseURL := strings.TrimRight(info.ChannelMeta.ChannelBaseUrl, "/") + apiKey := info.ChannelMeta.ApiKey + pollURL := fmt.Sprintf("%s/videos/%s", baseURL, videoID) + + sendProgress := func(msg string) { + delta := dto.ChatCompletionsStreamResponseChoiceDelta{} + delta.SetContentString(msg) + chunk := dto.ChatCompletionsStreamResponse{ + Id: "video-poll", + Object: "chat.completion.chunk", + Created: time.Now().Unix(), + Model: info.UpstreamModelName, + Choices: []dto.ChatCompletionsStreamResponseChoice{ + {Delta: delta}, + }, + } + _ = helper.ObjectData(c, chunk) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + client := &http.Client{Timeout: 30 * time.Second} + pollInterval := 3 * time.Second + prevStatus := "" + + for { + select { + case <-ctx.Done(): + return "", fmt.Errorf("video polling timed out") + case <-c.Request.Context().Done(): + return "", fmt.Errorf("client disconnected") + default: + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pollURL, nil) + if err != nil { + return "", fmt.Errorf("build poll request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+apiKey) + req.Header.Set("Accept", "application/json") + + pollResp, err := client.Do(req) + if err != nil { + logger.LogError(c, "video poll request error: "+err.Error()) + time.Sleep(pollInterval) + continue + } + + body, err := io.ReadAll(pollResp.Body) + pollResp.Body.Close() + if err != nil { + logger.LogError(c, "video poll read body error: "+err.Error()) + time.Sleep(pollInterval) + continue + } + + var status videoStatusResponse + if err := common.Unmarshal(body, &status); err != nil { + logger.LogError(c, "video poll unmarshal error: "+err.Error()) + time.Sleep(pollInterval) + continue + } + + if status.Status != prevStatus { + prevStatus = status.Status + switch status.Status { + case "queued": + sendProgress("⏳ 视频排队中...\n\n") + case "in_progress": + sendProgress("🎬 视频生成中...\n\n") + case "created": + sendProgress("📋 视频任务已创建...\n\n") + } + } + + if status.Status == "completed" { + // Return the backend proxy URL instead of the authenticated upstream URL, + // so the frontend can fetch the video without needing the Bearer token. + proxyURL := fmt.Sprintf("/pg/video/%d/%s/content", info.ChannelMeta.ChannelId, videoID) + return proxyURL, nil + } + if status.Status == "failed" { + errMsg := status.Error + if errMsg == "" { + errMsg = "unknown error" + } + return "", fmt.Errorf("video generation failed: %s", errMsg) + } + + time.Sleep(pollInterval) + } +} + +// handleVideoStreamData checks if lastStreamData is a video object and, if so, +// polls for completion and writes the final video SSE chunk. Returns true if +// video handling was performed (caller should skip normal last-response logic). +func handleVideoStreamData(c *gin.Context, info *relaycommon.RelayInfo, lastStreamData string) bool { + if lastStreamData == "" { + return false + } + var raw struct { + Object string `json:"object"` + ID string `json:"id"` + Status string `json:"status"` + } + if err := common.Unmarshal([]byte(lastStreamData), &raw); err != nil { + return false + } + if raw.Object != "video" && raw.Object != "video.generation" { + return false + } + if raw.ID == "" { + return false + } + + logger.LogInfo(c, fmt.Sprintf("video generation started, id=%s status=%s, polling...", raw.ID, raw.Status)) + + contentURL, err := pollVideoUntilComplete(c, info, raw.ID) + if err != nil { + logger.LogError(c, "video polling failed: "+err.Error()) + errDelta := dto.ChatCompletionsStreamResponseChoiceDelta{} + errDelta.SetContentString("❌ 视频生成失败: " + err.Error()) + chunk := dto.ChatCompletionsStreamResponse{ + Id: raw.ID, + Object: "chat.completion.chunk", + Created: time.Now().Unix(), + Model: info.UpstreamModelName, + Choices: []dto.ChatCompletionsStreamResponseChoice{ + {Delta: errDelta}, + }, + } + _ = helper.ObjectData(c, chunk) + helper.Done(c) + return true + } + + // Send video URL as a markdown video link so frontend renders an inline player. + videoMarkdown := fmt.Sprintf("[Generated Video](%s)", contentURL) + finishReason := "stop" + okDelta := dto.ChatCompletionsStreamResponseChoiceDelta{} + okDelta.SetContentString(videoMarkdown) + chunk := dto.ChatCompletionsStreamResponse{ + Id: raw.ID, + Object: "chat.completion.chunk", + Created: time.Now().Unix(), + Model: info.UpstreamModelName, + Choices: []dto.ChatCompletionsStreamResponseChoice{ + {Delta: okDelta, FinishReason: &finishReason}, + }, + } + _ = helper.ObjectData(c, chunk) + helper.Done(c) + return true +} + func OaiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) { if resp == nil || resp.Body == nil { logger.LogError(c, "invalid response or response body") @@ -162,15 +333,23 @@ func OaiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Re } } + // For video generation responses, poll until the video is ready and stream the URL. + // These are bare JSON objects (no data: prefix) forwarded by the scanner. + if info.IsPlayground && handleVideoStreamData(c, info, lastStreamData) { + return usage, nil + } + // 处理最后的响应 shouldSendLastResp := true - if err := handleLastResponse(lastStreamData, &responseId, &createAt, &systemFingerprint, &model, &usage, - &containStreamUsage, info, &shouldSendLastResp); err != nil { - logger.LogError(c, fmt.Sprintf("error handling last response: %s, lastStreamData: [%s]", err.Error(), lastStreamData)) + if lastStreamData != "" { + if err := handleLastResponse(lastStreamData, &responseId, &createAt, &systemFingerprint, &model, &usage, + &containStreamUsage, info, &shouldSendLastResp); err != nil { + logger.LogError(c, fmt.Sprintf("error handling last response: %s, lastStreamData: [%s]", err.Error(), lastStreamData)) + } } if info.RelayFormat == types.RelayFormatOpenAI { - if shouldSendLastResp { + if shouldSendLastResp && lastStreamData != "" { _ = sendStreamData(c, info, lastStreamData, info.ChannelSetting.ForceFormat, info.ChannelSetting.ThinkingToContent) } } @@ -557,6 +736,60 @@ func preConsumeUsage(ctx *gin.Context, info *relaycommon.RelayInfo, usage *dto.R return err } +// writePlaygroundImageSSE converts an image generation response into a proper +// SSE chat-completion stream so the playground frontend can display results. +func writePlaygroundImageSSE(c *gin.Context, responseBody []byte) { + helper.SetEventStreamHeaders(c) + + var imageResp dto.ImageResponse + if err := common.Unmarshal(responseBody, &imageResp); err != nil || len(imageResp.Data) == 0 { + helper.Done(c) + return + } + + var contentParts []string + for _, imgData := range imageResp.Data { + switch { + case imgData.B64Json != "": + contentParts = append(contentParts, "![Generated Image](data:image/png;base64,"+imgData.B64Json+")") + case imgData.Url != "": + contentParts = append(contentParts, "![Generated Image]("+imgData.Url+")") + } + } + if len(contentParts) == 0 { + helper.Done(c) + return + } + content := strings.Join(contentParts, "\n\n") + + contentChunk := dto.ChatCompletionsStreamResponse{ + Choices: []dto.ChatCompletionsStreamResponseChoice{{ + Index: 0, + Delta: dto.ChatCompletionsStreamResponseChoiceDelta{ + Role: "assistant", + Content: &content, + }, + }}, + } + if chunkBytes, err := common.Marshal(contentChunk); err == nil { + _ = helper.StringData(c, string(chunkBytes)) + } + + finishReason := "stop" + finishChunk := dto.ChatCompletionsStreamResponse{ + Choices: []dto.ChatCompletionsStreamResponseChoice{{ + Index: 0, + Delta: dto.ChatCompletionsStreamResponseChoiceDelta{}, + FinishReason: &finishReason, + }}, + } + if finishBytes, err := common.Marshal(finishChunk); err == nil { + _ = helper.StringData(c, string(finishBytes)) + } + + helper.Done(c) +} + func OpenaiHandlerWithUsage(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) { defer service.CloseResponseBodyGracefully(resp) @@ -571,8 +804,14 @@ func OpenaiHandlerWithUsage(c *gin.Context, info *relaycommon.RelayInfo, resp *h return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } - // 写入新的 response body - service.IOCopyBytesGracefully(c, resp, responseBody) + // For playground SSE image requests, wrap the response as a chat-completion + // stream so the frontend SSE client can parse and display it. + if info.IsPlayground && info.IsStream && info.RelayMode == relayconstant.RelayModeImagesGenerations { + writePlaygroundImageSSE(c, responseBody) + } else { + // 写入新的 response body + service.IOCopyBytesGracefully(c, resp, responseBody) + } // Once we've written to the client, we should not return errors anymore // because the upstream has already consumed resources and returned content diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index 2e157fc8cd91..2178cea254d0 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -151,6 +151,7 @@ type RelayInfo struct { RuntimeHeadersOverride map[string]interface{} UseRuntimeHeadersOverride bool ParamOverrideAudit []string + AudioDurationSeconds float64 // STT 响应中提取的音频时长（秒），用于按分钟计费 PriceData types.PriceData diff --git a/relay/compatible_handler.go b/relay/compatible_handler.go index 7a5624eb34d6..9afe0eccec30 100644 --- a/relay/compatible_handler.go +++ b/relay/compatible_handler.go @@ -23,7 +23,103 @@ import ( "github.com/gin-gonic/gin" ) +// extractImagePromptFromMessages returns the text content of the last user +// message, used as a prompt field for models that require it. +func extractImagePromptFromMessages(messages []dto.Message) string { + for i := len(messages) - 1; i >= 0; i-- { + if messages[i].Role == "user" { + return messages[i].StringContent() + } + } + return "" +} + +// isChatOnlyImageModel returns true for image-generation models that do not +// support the chat-completions endpoint (e.g. gpt-image-*, dall-e-*, +// chatgpt-image-*). Requests for these models received on /v1/chat/completions +// must be redirected to the images/generations handler. +func isChatOnlyImageModel(model string) bool { + lower := strings.ToLower(model) + return strings.Contains(lower, "gpt-image") || + strings.HasPrefix(lower, "dall-e") || + strings.Contains(lower, "chatgpt-image") +} + +// isVideoGenerationModel returns true for models that require a prompt field +// (video/image generation via chat-completions) but are not pure chat models. +func isVideoGenerationModel(model string) bool { + lower := strings.ToLower(model) + return strings.Contains(lower, "veo") || + strings.Contains(lower, "video") || + strings.Contains(lower, "generate") || + isChatOnlyImageModel(model) +} + +// isTTSModel returns true for text-to-speech models submitted via the chat +// completions endpoint from the playground. +func isTTSModel(model string) bool { + lower := strings.ToLower(model) + return strings.HasPrefix(lower, "tts") || + strings.Contains(lower, "speech") || + strings.Contains(lower, "text-to-speech") || + strings.Contains(lower, "cosyvoice") || + strings.Contains(lower, "sambert") +} + func TextHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *types.NewAPIError) { + // Detect image-generation models submitted to chat-completions endpoint and + // redirect to ImageHelper so the correct upstream endpoint and response + // handler are used. This avoids OaiStreamHandler receiving image JSON. + if chatReq, ok := info.Request.(*dto.GeneralOpenAIRequest); ok && isChatOnlyImageModel(chatReq.Model) { + prompt := extractImagePromptFromMessages(chatReq.Messages) + imageReq := &dto.ImageRequest{ + Model: chatReq.Model, + Prompt: prompt, + N: lo.ToPtr(uint(1)), + } + info.Request = imageReq + info.RelayMode = relayconstant.RelayModeImagesGenerations + info.RelayFormat = types.RelayFormatOpenAIImage + info.RequestURLPath = "/v1/images/generations" + return ImageHelper(c, info) + } + + // Detect TTS models from the playground and redirect to PlaygroundTTSHelper. + // Only applies to playground requests since the /v1/audio/speech endpoint handles + // direct API calls natively. + if info.IsPlayground { + if chatReq, ok := info.Request.(*dto.GeneralOpenAIRequest); ok && isTTSModel(chatReq.Model) { + // Re-read the body to pick up TTS-specific params (voice, speed, response_format) + // that don't exist on GeneralOpenAIRequest. + var ttsParams struct { + Voice string `json:"voice"` + Speed *float64 `json:"speed"` + ResponseFormat string `json:"response_format"` + } + _ = common.UnmarshalBodyReusable(c, &ttsParams) + + voice := ttsParams.Voice + if voice == "" { + voice = "alloy" + } + responseFormat := ttsParams.ResponseFormat + if responseFormat == "" { + responseFormat = "mp3" + } + + input := extractImagePromptFromMessages(chatReq.Messages) + audioReq := &dto.AudioRequest{ + Model: chatReq.Model, + Input: input, + Voice: voice, + ResponseFormat: responseFormat, + Speed: ttsParams.Speed, + } + info.Request = audioReq + return PlaygroundTTSHelper(c, info) + } + } + info.InitChannelMeta(c) textReq, ok := info.Request.(*dto.GeneralOpenAIRequest) @@ -45,6 +141,13 @@ func TextHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *types return types.NewError(err, types.ErrorCodeChannelModelMappedError, types.ErrOptionWithSkipRetry()) } + // For playground requests with video/image generation models, auto-populate + // prompt from the last user message when not already provided. + // Plain chat models do not use the prompt field. + if info.IsPlayground && request.Prompt == nil && isVideoGenerationModel(request.Model) { + request.Prompt = extractImagePromptFromMessages(request.Messages) + } + includeUsage := true // 判断用户是否需要返回使用情况 if request.StreamOptions != nil { diff --git a/relay/helper/price.go b/relay/helper/price.go index 8ba0ee8f0844..484850a7388e 100644 --- a/relay/helper/price.go +++ b/relay/helper/price.go @@ -76,6 +76,7 @@ func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens var cacheCreationRatio1h float64 var audioRatio float64 var audioCompletionRatio float64 + var audioMinutePrice float64 var freeModel bool if !usePrice { preConsumedTokens := common.Max(promptTokens, common.PreConsumedQuota) @@ -95,6 +96,15 @@ func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens } } completionRatio = ratio_setting.GetCompletionRatio(info.OriginModelName) + + // 分级定价：若配置了上下文分档，按 promptTokens 选择对应档位覆盖 modelRatio/completionRatio + if tiers, hasTier := ratio_setting.GetContextTierRatio(info.OriginModelName); hasTier { + if tier := types.SelectContextTier(tiers, promptTokens); tier != nil { + modelRatio = tier.InputRatio + completionRatio = tier.CompletionRatio + } + } + cacheRatio, _ = ratio_setting.GetCacheRatio(info.OriginModelName) cacheCreationRatio, _ = ratio_setting.GetCreateCacheRatio(info.OriginModelName) cacheCreationRatio5m = cacheCreationRatio @@ -103,6 +113,7 @@ func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens imageRatio, _ = ratio_setting.GetImageRatio(info.OriginModelName) audioRatio = ratio_setting.GetAudioRatio(info.OriginModelName) audioCompletionRatio = ratio_setting.GetAudioCompletionRatio(info.OriginModelName) + audioMinutePrice, _ = ratio_setting.GetAudioMinutePrice(info.OriginModelName) ratio := modelRatio * groupRatioInfo.GroupRatio preConsumedQuota = int(float64(preConsumedTokens) * ratio) } else { @@ -142,6 +153,7 @@ func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens ImageRatio: imageRatio, AudioRatio: audioRatio, AudioCompletionRatio: audioCompletionRatio, + AudioMinutePrice: audioMinutePrice, CacheCreationRatio: cacheCreationRatio, CacheCreation5mRatio: cacheCreationRatio5m, CacheCreation1hRatio: cacheCreationRatio1h, diff --git a/relay/helper/stream_scanner.go b/relay/helper/stream_scanner.go index a9bc5e16a720..3b00f991209d 100644 --- a/relay/helper/stream_scanner.go +++ b/relay/helper/stream_scanner.go @@ -245,6 +245,20 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon continue } if data[:5] != "data:" && data[:6] != "[DONE]" { + // Also forward bare JSON objects (e.g. video generation results + // returned without SSE "data:" framing by some providers). + trimmed := strings.TrimSpace(data) + if strings.HasPrefix(trimmed, "{") { + info.SetFirstResponseTime() + info.ReceivedResponseCount++ + select { + case dataChan <- trimmed: + case <-ctx.Done(): + return + case <-stopChan: + return + } + } continue } data = data[5:] diff --git a/relay/playground_audio.go b/relay/playground_audio.go new file mode 100644 index 000000000000..b3b41e4d2dd1 --- /dev/null +++ b/relay/playground_audio.go @@ -0,0 +1,234 @@ +package relay + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/logger" + relaycommon "github.com/QuantumNous/new-api/relay/common" + relayconstant "github.com/QuantumNous/new-api/relay/constant" + "github.com/QuantumNous/new-api/relay/helper" + "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +func audioDir() string { + dir := filepath.Join(os.TempDir(), "new-api-audio") + _ = os.MkdirAll(dir, 0755) + return dir +} + +func init() { + go func() { + ticker := time.NewTicker(30 * time.Minute) + for range ticker.C { + dir := audioDir() + entries, err := os.ReadDir(dir) + if err != nil { + continue + } + cutoff := time.Now().Add(-time.Hour) + for _, e := range entries { + if e.IsDir() { + continue + } + info, err := e.Info() + if err != nil { + continue + } + if info.ModTime().Before(cutoff) { + _ = os.Remove(filepath.Join(dir, e.Name())) + } + } + } + }() +} + +// GetCachedAudio retrieves a cached audio file by ID and extension. +// If ext is empty, it searches for any file matching the ID prefix. +func GetCachedAudio(id string) ([]byte, string, bool) { + dir := audioDir() + entries, err := os.ReadDir(dir) + if err != nil { + return nil, "", false + } + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + // match "uuid.ext" pattern + dotIdx := strings.LastIndex(name, ".") + if dotIdx == -1 { + continue + } + if name[:dotIdx] != id { + continue + } + ext := name[dotIdx:] + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return nil, "", false + } + return data, inferAudioContentType(ext[1:]), true // ext[1:] strips the leading "." + } + return nil, "", false +} + +// PlaygroundTTSHelper handles TTS model requests from the playground. +// It converts the chat-completion request into an audio/speech upstream call, +// persists the returned audio to disk, and emits an SSE chunk with a local proxy URL. +func PlaygroundTTSHelper(c *gin.Context, info *relaycommon.RelayInfo) *types.NewAPIError { + info.InitChannelMeta(c) + info.RelayMode = relayconstant.RelayModeAudioSpeech + info.RelayFormat = types.RelayFormatOpenAIAudio + info.RequestURLPath = "/v1/audio/speech" + info.IsStream = false // TTS upstream call is always non-streaming here + + audioReq, ok := info.Request.(*dto.AudioRequest) + if !ok { + return types.NewError(fmt.Errorf("invalid TTS request type"), types.ErrorCodeInvalidRequest, types.ErrOptionWithSkipRetry()) + } + + request, err := common.DeepCopy(audioReq) + if err != nil { + return types.NewError(fmt.Errorf("failed to copy audio request: %w", err), types.ErrorCodeInvalidRequest, types.ErrOptionWithSkipRetry()) + } + + err = helper.ModelMappedHelper(c, info, request) + if err != nil { + return types.NewError(err, types.ErrorCodeChannelModelMappedError, types.ErrOptionWithSkipRetry()) + } + + adaptor := GetAdaptor(info.ApiType) + if adaptor == nil { + return types.NewError(fmt.Errorf("invalid api type: %d", info.ApiType), types.ErrorCodeInvalidApiType, types.ErrOptionWithSkipRetry()) + } + adaptor.Init(info) + + ioReader, err := adaptor.ConvertAudioRequest(c, info, *request) + if err != nil { + return types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) + } + + resp, err := adaptor.DoRequest(c, info, ioReader) + if err != nil { + return types.NewError(err, types.ErrorCodeDoRequestFailed) + } + + httpResp, ok := resp.(*http.Response) + if !ok || httpResp == nil { + return types.NewError(fmt.Errorf("invalid upstream response"), types.ErrorCodeBadResponse, types.ErrOptionWithSkipRetry()) + } + defer service.CloseResponseBodyGracefully(httpResp) + + if httpResp.StatusCode != http.StatusOK { + return service.RelayErrorHandler(c.Request.Context(), httpResp, false) + } + + audioBytes, err := io.ReadAll(httpResp.Body) + if err != nil { + logger.LogError(c, "failed to read TTS response body: "+err.Error()) + return types.NewError(err, types.ErrorCodeReadResponseBodyFailed, types.ErrOptionWithSkipRetry()) + } + + contentType := httpResp.Header.Get("Content-Type") + if contentType == "" { + contentType = inferAudioContentType(request.ResponseFormat) + } + + audioID := uuid.New().String() + ext := audioExtFromContentType(contentType, request.ResponseFormat) + + // Persist to disk so the file survives server restarts. + filePath := filepath.Join(audioDir(), audioID+ext) + if err := os.WriteFile(filePath, audioBytes, 0644); err != nil { + logger.LogError(c, "failed to write TTS audio to disk: "+err.Error()) + return types.NewError(err, types.ErrorCodeReadResponseBodyFailed, types.ErrOptionWithSkipRetry()) + } + + // Consume quota (prompt tokens = input text length estimate). + usage := &dto.Usage{} + usage.PromptTokens = info.GetEstimatePromptTokens() + usage.TotalTokens = usage.PromptTokens + service.PostTextConsumeQuota(c, info, usage, nil) + + // Emit SSE chat-completion chunk with a markdown audio link. + audioURL := fmt.Sprintf("/pg/audio/%s%s", audioID, ext) + audioMarkdown := fmt.Sprintf("[Generated Audio](%s)", audioURL) + + helper.SetEventStreamHeaders(c) + finishReason := "stop" + delta := dto.ChatCompletionsStreamResponseChoiceDelta{} + delta.SetContentString(audioMarkdown) + chunk := dto.ChatCompletionsStreamResponse{ + Id: "tts-" + audioID, + Object: "chat.completion.chunk", + Created: time.Now().Unix(), + Model: info.UpstreamModelName, + Choices: []dto.ChatCompletionsStreamResponseChoice{ + {Delta: delta, FinishReason: &finishReason}, + }, + } + _ = helper.ObjectData(c, chunk) + helper.Done(c) + + return nil +} + +func inferAudioContentType(format string) string { + switch strings.ToLower(format) { + case "opus": + return "audio/ogg; codecs=opus" + case "aac": + return "audio/aac" + case "flac": + return "audio/flac" + case "wav": + return "audio/wav" + case "pcm": + return "audio/pcm" + default: + return "audio/mpeg" + } +} + +func audioExtFromContentType(contentType, format string) string { + ct := strings.ToLower(contentType) + if strings.Contains(ct, "ogg") || strings.Contains(ct, "opus") { + return ".opus" + } + if strings.Contains(ct, "aac") { + return ".aac" + } + if strings.Contains(ct, "flac") { + return ".flac" + } + if strings.Contains(ct, "wav") { + return ".wav" + } + if strings.Contains(ct, "pcm") { + return ".wav" + } + // fall back to format hint + switch strings.ToLower(format) { + case "opus": + return ".opus" + case "aac": + return ".aac" + case "flac": + return ".flac" + case "wav", "pcm": + return ".wav" + } + return ".mp3" +} diff --git a/router/relay-router.go b/router/relay-router.go index 17a13cad7fd6..f2b748788bf7 100644 --- a/router/relay-router.go +++ b/router/relay-router.go @@ -15,6 +15,7 @@ func SetRelayRouter(router *gin.Engine) { router.Use(middleware.DecompressRequestMiddleware()) router.Use(middleware.BodyStorageCleanup()) // 清理请求体存储 router.Use(middleware.StatsMiddleware()) + router.Use(middleware.RequestLogger()) // https://platform.openai.com/docs/api-reference/introduction modelsRouter := router.Group("/v1/models") modelsRouter.Use(middleware.RouteTag("relay")) @@ -66,6 +67,13 @@ func SetRelayRouter(router *gin.Engine) { { playgroundRouter.POST("/chat/completions", controller.Playground) } + + // Video and audio proxy routes — no auth required (IDs are unguessable) + pgVideoRouter := router.Group("/pg") + { + pgVideoRouter.GET("/video/:channelId/:videoId/content", controller.PlaygroundVideoProxy) + pgVideoRouter.GET("/audio/:audioId", controller.PlaygroundAudioProxy) + } relayV1Router := router.Group("/v1") relayV1Router.Use(middleware.RouteTag("relay")) relayV1Router.Use(middleware.SystemPerformanceCheck()) diff --git a/service/quota.go b/service/quota.go index 4150c44434bb..e3133d6e71f9 100644 --- a/service/quota.go +++ b/service/quota.go @@ -274,35 +274,47 @@ func PostAudioConsumeQuota(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, u groupRatio := relayInfo.PriceData.GroupRatioInfo.GroupRatio modelPrice := relayInfo.PriceData.ModelPrice usePrice := relayInfo.PriceData.UsePrice + audioMinutePrice := relayInfo.PriceData.AudioMinutePrice - quotaInfo := QuotaInfo{ - InputDetails: TokenDetails{ - TextTokens: textInputTokens, - AudioTokens: audioInputTokens, - }, - OutputDetails: TokenDetails{ - TextTokens: textOutTokens, - AudioTokens: audioOutTokens, - }, - ModelName: relayInfo.OriginModelName, - UsePrice: usePrice, - ModelRatio: modelRatio, - GroupRatio: groupRatio, - } - - quota := calculateAudioQuota(quotaInfo) - - totalTokens := usage.TotalTokens + var quota int var logContent string - if !usePrice { - logContent = fmt.Sprintf("模型倍率 %.2f，补全倍率 %.2f，音频倍率 %.2f，音频补全倍率 %.2f，分组倍率 %.2f", - modelRatio, completionRatio.InexactFloat64(), audioRatio.InexactFloat64(), audioCompletionRatio.InexactFloat64(), groupRatio) + + // 按音频时长（分钟）计费：优先级高于 token 计费 + if audioMinutePrice > 0 && relayInfo.AudioDurationSeconds > 0 { + durationMinutes := math.Ceil(relayInfo.AudioDurationSeconds / 60.0) + quotaDecimal := decimal.NewFromFloat(durationMinutes). + Mul(decimal.NewFromFloat(audioMinutePrice)). + Mul(decimal.NewFromFloat(common.QuotaPerUnit)). + Mul(decimal.NewFromFloat(groupRatio)) + quota = int(quotaDecimal.Round(0).IntPart()) + logContent = fmt.Sprintf("按时长计费：%.1f 秒（%.0f 分钟），每分钟 $%.4f，分组倍率 %.2f", + relayInfo.AudioDurationSeconds, durationMinutes, audioMinutePrice, groupRatio) } else { - logContent = fmt.Sprintf("模型价格 %.2f，分组倍率 %.2f", modelPrice, groupRatio) + quotaInfo := QuotaInfo{ + InputDetails: TokenDetails{ + TextTokens: textInputTokens, + AudioTokens: audioInputTokens, + }, + OutputDetails: TokenDetails{ + TextTokens: textOutTokens, + AudioTokens: audioOutTokens, + }, + ModelName: relayInfo.OriginModelName, + UsePrice: usePrice, + ModelRatio: modelRatio, + GroupRatio: groupRatio, + } + quota = calculateAudioQuota(quotaInfo) + if !usePrice { + logContent = fmt.Sprintf("模型倍率 %.2f，补全倍率 %.2f，音频倍率 %.2f，音频补全倍率 %.2f，分组倍率 %.2f", + modelRatio, completionRatio.InexactFloat64(), audioRatio.InexactFloat64(), audioCompletionRatio.InexactFloat64(), groupRatio) + } else { + logContent = fmt.Sprintf("模型价格 %.2f，分组倍率 %.2f", modelPrice, groupRatio) + } } // record all the consume log even if quota is 0 - if totalTokens == 0 { + if usage.TotalTokens == 0 && relayInfo.AudioDurationSeconds == 0 { // in this case, must be some error happened // we cannot just return, because we may have to return the pre-consumed quota quota = 0 diff --git a/setting/ratio_setting/model_ratio.go b/setting/ratio_setting/model_ratio.go index 7556fd9482c7..5ff147db2cc7 100644 --- a/setting/ratio_setting/model_ratio.go +++ b/setting/ratio_setting/model_ratio.go @@ -349,6 +349,8 @@ func InitRatioSettings() { imageRatioMap.AddAll(defaultImageRatio) audioRatioMap.AddAll(defaultAudioRatio) audioCompletionRatioMap.AddAll(defaultAudioCompletionRatio) + contextTierRatioMap.AddAll(defaultContextTierRatio) + audioMinutePriceMap.AddAll(defaultAudioMinutePrice) } func GetModelPriceMap() map[string]float64 { @@ -738,3 +740,54 @@ func GetModelRatioOrPrice(model string) (float64, bool, bool) { // price or rati } return 37.5, false, false } + +// ContextTierRatio — 按上下文长度分档定价 +// key: 模型名, value: 有序 tier 列表（从小到大排列 MaxTokens，最后一项 MaxTokens=-1 兜底） +// input_ratio 对应 ModelRatio，completion_ratio 对应 CompletionRatio +var defaultContextTierRatio = map[string][]types.ContextTierRatio{ + // qwen3-max: 32K / 32-128K / 128-252K 三档 + "qwen3-max": { + {MaxTokens: 32768, InputRatio: 2.5, CompletionRatio: 2}, + {MaxTokens: 131072, InputRatio: 2.0, CompletionRatio: 4}, + {MaxTokens: -1, InputRatio: 3.5, CompletionRatio: 4}, + }, +} + +var contextTierRatioMap = types.NewRWMap[string, []types.ContextTierRatio]() + +func ContextTierRatio2JSONString() string { + return contextTierRatioMap.MarshalJSONString() +} + +func UpdateContextTierRatioByJSONString(jsonStr string) error { + return types.LoadFromJsonStringWithCallback(contextTierRatioMap, jsonStr, InvalidateExposedDataCache) +} + +func GetContextTierRatio(name string) ([]types.ContextTierRatio, bool) { + name = FormatMatchingModelName(name) + tiers, ok := contextTierRatioMap.Get(name) + return tiers, ok +} + +// AudioMinutePrice — 按音频时长（分钟）计费，适用于 STT 模型（如 whisper-1） +// key: 模型名, value: 每分钟音频的价格（美元） +var defaultAudioMinutePrice = map[string]float64{ + "whisper-1": 0.006, // $0.006 per minute +} + +var audioMinutePriceMap = types.NewRWMap[string, float64]() + +func AudioMinutePrice2JSONString() string { + return audioMinutePriceMap.MarshalJSONString() +} + +func UpdateAudioMinutePriceByJSONString(jsonStr string) error { + return types.LoadFromJsonStringWithCallback(audioMinutePriceMap, jsonStr, InvalidateExposedDataCache) +} + +// GetAudioMinutePrice 返回模型的每分钟音频价格（美元），若未配置返回 0, false +func GetAudioMinutePrice(name string) (float64, bool) { + name = FormatMatchingModelName(name) + price, ok := audioMinutePriceMap.Get(name) + return price, ok +} diff --git a/types/context_tier.go b/types/context_tier.go new file mode 100644 index 000000000000..7033f408273b --- /dev/null +++ b/types/context_tier.go @@ -0,0 +1,24 @@ +package types + +// ContextTierRatio defines a pricing tier based on prompt token count. +// Tiers are evaluated in order; the first matching tier is used. +// Set MaxTokens to -1 on the last tier to act as the catch-all. +type ContextTierRatio struct { + MaxTokens int `json:"max_tokens"` // inclusive upper bound; -1 = unlimited + InputRatio float64 `json:"input_ratio"` // replaces ModelRatio for this tier + CompletionRatio float64 `json:"completion_ratio"` // replaces CompletionRatio for this tier +} + +// SelectContextTier returns the tier that matches the given prompt token count. +// Returns nil if the slice is empty. +func SelectContextTier(tiers []ContextTierRatio, promptTokens int) *ContextTierRatio { + for i := range tiers { + if tiers[i].MaxTokens == -1 || promptTokens <= tiers[i].MaxTokens { + return &tiers[i] + } + } + if len(tiers) > 0 { + return &tiers[len(tiers)-1] + } + return nil +} diff --git a/types/price_data.go b/types/price_data.go index 93bc6ae8d168..7672b58c5ab6 100644 --- a/types/price_data.go +++ b/types/price_data.go @@ -20,6 +20,7 @@ type PriceData struct { ImageRatio float64 AudioRatio float64 AudioCompletionRatio float64 + AudioMinutePrice float64 // >0 时对 STT 模型按分钟计费（美元/分钟），覆盖 token 计费 OtherRatios map[string]float64 UsePrice bool Quota int // 按次计费的最终额度（MJ / Task） diff --git a/web/bun.lock b/web/bun.lock index da3c1e452a9d..4f109c348221 100644 --- a/web/bun.lock +++ b/web/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "react-template", diff --git a/web/src/components/common/markdown/MarkdownRenderer.jsx b/web/src/components/common/markdown/MarkdownRenderer.jsx index 6a71c695f845..e65bd620f213 100644 --- a/web/src/components/common/markdown/MarkdownRenderer.jsx +++ b/web/src/components/common/markdown/MarkdownRenderer.jsx @@ -42,6 +42,67 @@ mermaid.initialize({ securityLevel: 'loose', }); +function AudioPlayer({ src, title }) { + const { t } = useTranslation(); + const [loadError, setLoadError] = useState(false); + + if (loadError) { + return ( +
+ {t('音频加载失败')} +
+ ); + } + + return ( +
+ {title && ( +
+ {title} +
+ )} +
+ ); +} + export function Mermaid(props) { const ref = useRef(null); const [hasError, setHasError] = useState(false); @@ -414,9 +475,24 @@ function _MarkdownContent(props) { url} components={{ pre: PreCode, code: CustomCode, + img: ({ src, alt, ...imgProps }) => ( + {alt + ), p: (pProps) => (

{ const href = aProps.href || ''; - if (/\.(aac|mp3|opus|wav)$/.test(href)) { - return ( -

- -
- ); + const hrefPath = href.split('?')[0].split('#')[0]; + if (/\.(aac|mp3|opus|wav)$/.test(hrefPath)) { + const title = + typeof aProps.children === 'string' && + aProps.children !== href + ? aProps.children + : null; + return ; } - if (/\.(3gp|3g2|webm|ogv|mpeg|mp4|avi)$/.test(href)) { + const isVideoUrl = + /\.(3gp|3g2|webm|ogv|mpeg|mp4|avi)$/.test(hrefPath) || + /\/videos?\/[^/].*\/content$/.test(hrefPath); + if (isVideoUrl) { return ( ); } diff --git a/web/src/hooks/playground/useApiRequest.jsx b/web/src/hooks/playground/useApiRequest.jsx index f072084b63ae..97d5f35f2e06 100644 --- a/web/src/hooks/playground/useApiRequest.jsx +++ b/web/src/hooks/playground/useApiRequest.jsx @@ -32,6 +32,36 @@ import { processIncompleteThinkTags, } from '../../helpers'; +// formatVideoResponse converts a raw video-generation object into markdown content. +// Returns null when the payload contains no useful information. +function formatVideoResponse(payload) { + if (!payload) return null; + + // Try to find a video URL in common locations + const url = + payload.url || + payload.video_url || + payload.data?.[0]?.url || + payload.data?.[0]?.video_url; + + const parts = []; + if (url) { + parts.push(`[Generated Video](${url})`); + } + + const meta = []; + if (payload.model) meta.push(`模型: ${payload.model}`); + if (payload.size) meta.push(`分辨率: ${payload.size}`); + if (payload.seconds != null) meta.push(`时长: ${payload.seconds}s`); + if (payload.status) meta.push(`状态: ${payload.status}`); + + if (meta.length > 0) { + parts.push(`**视频生成完成** — ${meta.join(' | ')}`); + } + + return parts.length > 0 ? parts.join('\n\n') : null; +} + export const useApiRequest = ( setMessage, setDebugData, @@ -370,6 +400,14 @@ export const useApiRequest = ( streamMessageUpdate(delta.content, 'content'); } } + + // Handle video generation responses (object: "video") + if (payload.object === 'video' || payload.object === 'video.generation') { + const videoContent = formatVideoResponse(payload); + if (videoContent) { + streamMessageUpdate(videoContent, 'content'); + } + } } catch (error) { console.error('Failed to parse SSE message:', error); const errorInfo = `解析错误: ${error.message}`; diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index 7e4db5f36563..ae0f42c4d7b4 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -3673,6 +3673,7 @@ "默认折叠侧边栏": "Default collapse sidebar", "默认测试模型": "Default Test Model", "默认用户消息": "Default User Message", - "默认补全倍率": "Default completion ratio" + "默认补全倍率": "Default completion ratio", + "音频加载失败": "Failed to load audio" } } diff --git a/web/src/i18n/locales/zh-CN.json b/web/src/i18n/locales/zh-CN.json index 8c52cdfbba75..07ee12d14a11 100644 --- a/web/src/i18n/locales/zh-CN.json +++ b/web/src/i18n/locales/zh-CN.json @@ -1662,8 +1662,8 @@ "操作确认": "操作确认", "操作管理员": "操作管理员", "操作类型": "操作类型", - "操练场": "操练场", - "操练场和聊天功能": "操练场和聊天功能", + "操练场": "话题", + "操练场和聊天功能": "话题和聊天功能", "支付": "支付", "支付地址": "支付地址", "支付失败": "支付失败", diff --git a/web/src/pages/Setting/Ratio/ModelRatioSettings.jsx b/web/src/pages/Setting/Ratio/ModelRatioSettings.jsx index e9be19785547..68064e2ec8ff 100644 --- a/web/src/pages/Setting/Ratio/ModelRatioSettings.jsx +++ b/web/src/pages/Setting/Ratio/ModelRatioSettings.jsx @@ -48,6 +48,8 @@ export default function ModelRatioSettings(props) { ImageRatio: '', AudioRatio: '', AudioCompletionRatio: '', + ContextTierRatio: '', + AudioMinutePrice: '', ExposeRatioEnabled: false, }); const refForm = useRef(); @@ -319,6 +321,58 @@ export default function ModelRatioSettings(props) { /> + + + verifyJSON(value), + message: '不是合法的 JSON 字符串', + }, + ]} + onChange={(value) => + setInputs({ ...inputs, ContextTierRatio: value }) + } + /> + + + + + verifyJSON(value), + message: '不是合法的 JSON 字符串', + }, + ]} + onChange={(value) => + setInputs({ ...inputs, AudioMinutePrice: value }) + } + /> + +