Add doubao asr｜增加豆包语音识别模型支持

docs/openapi/relay.json

@@ -1595,7 +1595,7 @@
       "post": {
         "summary": "音频转录",
         "deprecated": false,
-        "description": "将音频转换为文本",
+        "description": "将音频转换为文本。兼容 OpenAI Whisper 请求格式，也可通过 VolcEngine/Doubao 语音渠道转发到豆包 ASR。",
         "operationId": "createTranscription",
         "tags": [
           "OpenAI音频(Audio)"

relay/channel/volcengine/adaptor.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/QuantumNous/new-api/common"
 	channelconstant "github.com/QuantumNous/new-api/constant"
 	"github.com/QuantumNous/new-api/dto"
 	"github.com/QuantumNous/new-api/relay/channel"
@@ -47,6 +48,19 @@ func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayIn
 }
 
 func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.AudioRequest) (io.Reader, error) {
+	if info.RelayMode == constant.RelayModeAudioTranscription {
+		c.Set(contextKeyResponseFormat, request.ResponseFormat)
+		volcRequest, err := buildASRRequest(c, info)
+		if err != nil {
+			return nil, err
+		}
+		jsonData, err := common.Marshal(volcRequest)
+		if err != nil {
+			return nil, fmt.Errorf("error marshalling volcengine asr request: %w", err)
+		}
+		return bytes.NewReader(jsonData), nil
+	}
+
 	if info.RelayMode != constant.RelayModeAudioSpeech {
 		return nil, errors.New("unsupported audio relay mode")
 	}
@@ -86,7 +100,7 @@ func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInf
 	}
 
 	if len(request.Metadata) > 0 {
-		if err = json.Unmarshal(request.Metadata, &volcRequest); err != nil {
+		if err = common.Unmarshal(request.Metadata, &volcRequest); err != nil {
 			return nil, fmt.Errorf("error unmarshalling metadata to volcengine request: %w", err)
 		}
 	}
@@ -97,7 +111,7 @@ func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInf
 		info.IsStream = true
 	}
 
-	jsonData, err := json.Marshal(volcRequest)
+	jsonData, err := common.Marshal(volcRequest)
 	if err != nil {
 		return nil, fmt.Errorf("error marshalling volcengine request: %w", err)
 	}
@@ -273,6 +287,8 @@ func (a *Adaptor) GetRequestURL(info *relaycommon.RelayInfo) (string, error) {
 			return fmt.Sprintf("%s/api/v3/rerank", baseUrl), nil
 		case constant.RelayModeResponses:
 			return fmt.Sprintf("%s/api/v3/responses", baseUrl), nil
+		case constant.RelayModeAudioTranscription:
+			return volcengineASRURL, nil
 		case constant.RelayModeAudioSpeech:
 			if baseUrl == channelconstant.ChannelBaseURLs[channelconstant.ChannelTypeVolcEngine] {
 				return "wss://openspeech.bytedance.com/api/v1/tts/ws_binary", nil
@@ -294,6 +310,19 @@ func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Header, info *rel
 		}
 		req.Set("Content-Type", "application/json")
 		return nil
+	} else if info.RelayMode == constant.RelayModeAudioTranscription {
+		appID, token, err := parseVolcengineAuth(info.ApiKey)
+		if err != nil {
+			return err
+		}
+		req.Set("X-Api-App-Key", appID)
+		req.Set("X-Api-Access-Key", token)
+		req.Set("X-Api-Resource-Id", volcengineASRResourceID)
+		req.Set("X-Api-Request-Id", newVolcengineRequestID())
+		req.Set("X-Api-Sequence", "-1")
+		req.Set("Content-Type", gin.MIMEJSON)
+		req.Set("Accept", gin.MIMEJSON)
+		return nil
 	} else if info.RelayMode == constant.RelayModeImagesEdits {
 		req.Set("Content-Type", gin.MIMEJSON)
 	}
@@ -388,6 +417,10 @@ func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycom
 		return handleTTSResponse(c, resp, info, encoding)
 	}
 
+	if info.RelayMode == constant.RelayModeAudioTranscription {
+		return handleASRResponse(c, resp, info, c.GetString(contextKeyResponseFormat))
+	}
+
 	adaptor := openai.Adaptor{}
 	usage, err = adaptor.DoResponse(c, resp, info)
 	return

relay/channel/volcengine/asr.go

@@ -0,0 +1,215 @@
+package volcengine
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/QuantumNous/new-api/dto"
+	relaycommon "github.com/QuantumNous/new-api/relay/common"
+	"github.com/QuantumNous/new-api/types"
+	"github.com/gin-gonic/gin"
+	"github.com/google/uuid"
+)
+
+const (
+	volcengineASRURL        = "https://openspeech.bytedance.com/api/v3/auc/bigmodel/recognize/flash"
+	volcengineASRResourceID = "volc.bigasr.auc_turbo"
+)
+
+type VolcengineASRRequest struct {
+	User    VolcengineASRUser    `json:"user"`
+	Audio   VolcengineASRAudio   `json:"audio"`
+	Request VolcengineASRReqInfo `json:"request"`
+}
+
+type VolcengineASRUser struct {
+	UID string `json:"uid"`
+}
+
+type VolcengineASRAudio struct {
+	Data   string `json:"data"`
+	Format string `json:"format,omitempty"`
+}
+
+type VolcengineASRReqInfo struct {
+	ModelName      string `json:"model_name"`
+	EnableITN      bool   `json:"enable_itn"`
+	EnablePunc     bool   `json:"enable_punc"`
+	EnableDDC      bool   `json:"enable_ddc"`
+	ShowUtterances bool   `json:"show_utterances,omitempty"`
+}
+
+type VolcengineASRResponse struct {
+	Code    int                  `json:"code"`
+	Message string               `json:"message"`
+	Result  *VolcengineASRResult `json:"result"`
+}
+
+type VolcengineASRResult struct {
+	Text string `json:"text"`
+}
+
+func buildASRRequest(c *gin.Context, info *relaycommon.RelayInfo) (*VolcengineASRRequest, error) {
+	appID, _, err := parseVolcengineAuth(info.ApiKey)
+	if err != nil {
+		return nil, err
+	}
+
+	formData, err := common.ParseMultipartFormReusable(c)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing multipart form: %w", err)
+	}
+
+	fileHeaders := formData.File["file"]
+	if len(fileHeaders) == 0 {
+		return nil, errors.New("file is required")
+	}
+
+	fileHeader := fileHeaders[0]
+	file, err := fileHeader.Open()
+	if err != nil {
+		return nil, fmt.Errorf("error opening audio file: %w", err)
+	}
+	defer file.Close()
+
+	audioData, err := io.ReadAll(file)
+	if err != nil {
+		return nil, fmt.Errorf("error reading audio file: %w", err)
+	}
+
+	request := &VolcengineASRRequest{
+		User: VolcengineASRUser{
+			UID: appID,
+		},
+		Audio: VolcengineASRAudio{
+			Data:   base64.StdEncoding.EncodeToString(audioData),
+			Format: detectAudioFormat(fileHeader.Filename, fileHeader.Header.Get("Content-Type")),
+		},
+		Request: VolcengineASRReqInfo{
+			ModelName:      info.UpstreamModelName,
+			EnableITN:      true,
+			EnablePunc:     true,
+			EnableDDC:      true,
+			ShowUtterances: true,
+		},
+	}
+
+	if request.Request.ModelName == "" {
+		request.Request.ModelName = info.OriginModelName
+	}
+
+	return request, nil
+}
+
+func detectAudioFormat(filename string, contentType string) string {
+	ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".")
+	switch ext {
+	case "wav", "mp3", "ogg", "opus", "m4a", "mp4", "aac", "flac":
+		if ext == "opus" {
+			return "ogg"
+		}
+		return ext
+	}
+
+	contentType = strings.ToLower(contentType)
+	switch {
+	case strings.Contains(contentType, "wav"):
+		return "wav"
+	case strings.Contains(contentType, "mpeg"), strings.Contains(contentType, "mp3"):
+		return "mp3"
+	case strings.Contains(contentType, "ogg"), strings.Contains(contentType, "opus"):
+		return "ogg"
+	default:
+		return ""
+	}
+}
+
+func handleASRResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo, responseFormat string) (usage any, err *types.NewAPIError) {
+	defer resp.Body.Close()
+
+	body, readErr := io.ReadAll(resp.Body)
+	if readErr != nil {
+		return nil, types.NewErrorWithStatusCode(
+			errors.New("failed to read volcengine asr response"),
+			types.ErrorCodeReadResponseBodyFailed,
+			http.StatusInternalServerError,
+		)
+	}
+
+	var volcResp VolcengineASRResponse
+	if unmarshalErr := common.Unmarshal(body, &volcResp); unmarshalErr != nil {
+		return nil, types.NewErrorWithStatusCode(
+			errors.New("failed to parse volcengine asr response"),
+			types.ErrorCodeBadResponseBody,
+			http.StatusInternalServerError,
+		)
+	}
+
+	statusCode := strings.TrimSpace(resp.Header.Get("X-Api-Status-Code"))
+	if statusCode == "" && volcResp.Code != 0 {
+		statusCode = fmt.Sprintf("%d", volcResp.Code)
+	}
+	if statusCode != "" && statusCode != "20000000" {
+		message := strings.TrimSpace(resp.Header.Get("X-Api-Message"))
+		if message == "" {
+			message = strings.TrimSpace(volcResp.Message)
+		}
+		if message == "" {
+			message = "volcengine asr request failed"
+		}
+		return nil, types.NewErrorWithStatusCode(
+			errors.New(message),
+			types.ErrorCodeBadResponse,
+			http.StatusBadRequest,
+		)
+	}
+
+	transcript := ""
+	if volcResp.Result != nil {
+		transcript = volcResp.Result.Text
+	}
+
+	// VolcEngine flash ASR currently maps safely to OpenAI-style `json` and `text`
+	// outputs only. `srt` and `vtt` are not generated here.
+	switch responseFormat {
+	case "", "json", "verbose_json":
+		payload, marshalErr := common.Marshal(dto.AudioResponse{Text: transcript})
+		if marshalErr != nil {
+			return nil, types.NewErrorWithStatusCode(
+				marshalErr,
+				types.ErrorCodeBadResponseBody,
+				http.StatusInternalServerError,
+			)
+		}
+		c.Header("Content-Type", gin.MIMEJSON)
+		c.Status(http.StatusOK)
+		_, _ = c.Writer.Write(payload)
+	case "text":
+		c.Header("Content-Type", "text/plain; charset=utf-8")
+		c.Status(http.StatusOK)
+		_, _ = io.Copy(c.Writer, bytes.NewBufferString(transcript))
+	default:
+		return nil, types.NewErrorWithStatusCode(
+			fmt.Errorf("unsupported response_format for volcengine asr: %s (unsupported formats include srt and vtt)", responseFormat),
+			types.ErrorCodeInvalidRequest,
+			http.StatusBadRequest,
+		)
+	}
+
+	estimate := info.GetEstimatePromptTokens()
+	return &dto.Usage{
+		PromptTokens: estimate,
+		TotalTokens:  estimate,
+	}, nil
+}
+
+func newVolcengineRequestID() string {
+	return uuid.NewString()
+}

relay/channel/volcengine/constants.go

@@ -1,6 +1,7 @@
 package volcengine
 
 var ModelList = []string{
+	"bigmodel",
 	"Doubao-pro-128k",
 	"Doubao-pro-32k",
 	"Doubao-pro-4k",