diff --git a/dto/gemini.go b/dto/gemini.go index 489ebea534b4..69b869974a2a 100644 --- a/dto/gemini.go +++ b/dto/gemini.go @@ -22,6 +22,97 @@ type GeminiChatRequest struct { CachedContent string `json:"cachedContent,omitempty"` } +// GeminiInteractionRequest carries native Vertex interaction payloads such as +// Lyria 3 requests without forcing them into generateContent semantics. +type GeminiInteractionRequest struct { + Model *string `json:"model,omitempty"` + Payload map[string]any `json:"-"` +} + +// UnmarshalJSON preserves the complete native interaction payload while +// exposing model for channel model mapping and billing. +func (r *GeminiInteractionRequest) UnmarshalJSON(data []byte) error { + var payload map[string]any + if err := common.Unmarshal(data, &payload); err != nil { + return err + } + r.Payload = payload + if model, ok := payload["model"].(string); ok { + r.Model = &model + } + return nil +} + +// MarshalJSON writes the mapped model back into the preserved native payload. +func (r GeminiInteractionRequest) MarshalJSON() ([]byte, error) { + payload := make(map[string]any, len(r.Payload)+1) + for k, v := range r.Payload { + payload[k] = v + } + if r.Model != nil { + payload["model"] = *r.Model + } + return common.Marshal(payload) +} + +// GetTokenCountMeta estimates billing text from the native interaction input. +func (r *GeminiInteractionRequest) GetTokenCountMeta() *types.TokenCountMeta { + var inputTexts []string + collectGeminiInteractionTexts(r.Payload["input"], &inputTexts) + return &types.TokenCountMeta{ + CombineText: strings.Join(inputTexts, "\n"), + } +} + +// IsStream returns false because Vertex interactions are non-streaming. +func (r *GeminiInteractionRequest) IsStream(c *gin.Context) bool { + return false +} + +// SetModelName keeps the URL-selected or mapped model in the native payload. +func (r *GeminiInteractionRequest) SetModelName(modelName string) { + if modelName == "" { + return + } + r.Model = &modelName + if r.Payload == nil { + r.Payload = map[string]any{} + } + r.Payload["model"] = modelName +} + +// collectGeminiInteractionTexts extracts text from flexible interaction input +// arrays without interpreting provider-specific media fields. +func collectGeminiInteractionTexts(value any, texts *[]string) { + switch v := value.(type) { + case string: + if v != "" { + *texts = append(*texts, v) + } + case []any: + for _, item := range v { + collectGeminiInteractionTexts(item, texts) + } + case map[string]any: + if itemType, ok := v["type"].(string); ok { + if itemType != "text" { + return + } + if text, ok := v["text"].(string); ok && text != "" { + *texts = append(*texts, text) + } + return + } + if text, ok := v["text"].(string); ok && text != "" { + *texts = append(*texts, text) + return + } + for _, item := range v { + collectGeminiInteractionTexts(item, texts) + } + } +} + // UnmarshalJSON allows GeminiChatRequest to accept both snake_case and camelCase fields. func (r *GeminiChatRequest) UnmarshalJSON(data []byte) error { type Alias GeminiChatRequest @@ -44,9 +135,9 @@ func (r *GeminiChatRequest) UnmarshalJSON(data []byte) error { } type ToolConfig struct { - FunctionCallingConfig *FunctionCallingConfig `json:"functionCallingConfig,omitempty"` - RetrievalConfig *RetrievalConfig `json:"retrievalConfig,omitempty"` - IncludeServerSideToolInvocations *bool `json:"includeServerSideToolInvocations,omitempty"` + FunctionCallingConfig *FunctionCallingConfig `json:"functionCallingConfig,omitempty"` + RetrievalConfig *RetrievalConfig `json:"retrievalConfig,omitempty"` + IncludeServerSideToolInvocations *bool `json:"includeServerSideToolInvocations,omitempty"` } type FunctionCallingConfig struct { diff --git a/relay/channel/vertex/adaptor.go b/relay/channel/vertex/adaptor.go index 0d91032d0f35..d02229720454 100644 --- a/relay/channel/vertex/adaptor.go +++ b/relay/channel/vertex/adaptor.go @@ -167,6 +167,22 @@ func (a *Adaptor) getRequestUrl(info *relaycommon.RelayInfo, modelName, suffix s } func (a *Adaptor) GetRequestURL(info *relaycommon.RelayInfo) (string, error) { + if a.isLyriaInteractionRequest(info) { + if info.ChannelOtherSettings.VertexKeyType == dto.VertexKeyTypeAPIKey { + return fmt.Sprintf( + "%s?key=%s", + BuildGoogleModelURL(info.ChannelBaseUrl, OpenSourceAPIVersion, "", "global", info.UpstreamModelName, "interactions"), + info.ApiKey, + ), nil + } + adc := &Credentials{} + if err := common.Unmarshal([]byte(info.ApiKey), adc); err != nil { + return "", fmt.Errorf("failed to decode credentials file: %w", err) + } + a.AccountCredentials = *adc + return fmt.Sprintf("%s/interactions", BuildAPIBaseURL(info.ChannelBaseUrl, OpenSourceAPIVersion, adc.ProjectID, "global")), nil + } + suffix := "" if a.RequestMode == RequestModeGemini { if model_setting.GetGeminiSettings().ThinkingAdapterEnabled && @@ -328,6 +344,9 @@ func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, request } func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (usage any, err *types.NewAPIError) { + if a.isLyriaInteractionRequest(info) { + return a.doInteractionResponse(c, resp, info) + } claudeAdaptor := claude.Adaptor{} if info.IsStream { switch a.RequestMode { @@ -362,6 +381,54 @@ func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycom return } +// isLyriaInteractionRequest identifies the native Vertex interactions endpoint +// used by Lyria 3 models on the Gemini relay path. +func (a *Adaptor) isLyriaInteractionRequest(info *relaycommon.RelayInfo) bool { + if info == nil { + return false + } + requestPath := strings.Split(info.RequestURLPath, "?")[0] + return info.RelayMode == constant.RelayModeGemini && + strings.Contains(requestPath, ":interactions") && + strings.HasPrefix(info.UpstreamModelName, "lyria-3-") +} + +// doInteractionResponse preserves the native Vertex interaction JSON so callers +// can read all text, metadata, status, and audio outputs. +func (a *Adaptor) doInteractionResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (*dto.Usage, *types.NewAPIError) { + defer resp.Body.Close() + bodyBytes, readErr := io.ReadAll(resp.Body) + if readErr != nil { + return nil, types.NewErrorWithStatusCode(readErr, types.ErrorCodeReadResponseBodyFailed, http.StatusInternalServerError) + } + bodySnippet := string(bodyBytes) + if len(bodySnippet) > 512 { + bodySnippet = bodySnippet[:512] + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, types.NewErrorWithStatusCode( + fmt.Errorf("vertex upstream status=%d body=%s", resp.StatusCode, bodySnippet), + types.ErrorCodeBadResponseBody, + resp.StatusCode, + ) + } + + modelName := strings.TrimSpace(info.UpstreamModelName) + if strings.HasPrefix(modelName, "lyria-3-") { + contentType := resp.Header.Get("Content-Type") + if strings.TrimSpace(contentType) == "" { + contentType = "application/json" + } + c.Data(http.StatusOK, contentType, bodyBytes) + return &dto.Usage{ + PromptTokens: 1, + TotalTokens: 1, + }, nil + } + + return nil, types.NewErrorWithStatusCode(fmt.Errorf("unsupported vertex interaction model: %s", modelName), types.ErrorCodeInvalidRequest, http.StatusBadRequest) +} + func (a *Adaptor) GetModelList() []string { var modelList []string for i, s := range ModelList { diff --git a/relay/gemini_handler.go b/relay/gemini_handler.go index 3b4bafe2a673..c22b87f74be5 100644 --- a/relay/gemini_handler.go +++ b/relay/gemini_handler.go @@ -55,6 +55,10 @@ func trimModelThinking(modelName string) string { func GeminiHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *types.NewAPIError) { info.InitChannelMeta(c) + if interactionReq, ok := info.Request.(*dto.GeminiInteractionRequest); ok { + return GeminiInteractionHelper(c, info, interactionReq) + } + geminiReq, ok := info.Request.(*dto.GeminiChatRequest) if !ok { return types.NewErrorWithStatusCode(fmt.Errorf("invalid request type, expected *dto.GeminiChatRequest, got %T", info.Request), types.ErrorCodeInvalidRequest, http.StatusBadRequest, types.ErrOptionWithSkipRetry()) @@ -198,6 +202,66 @@ func GeminiHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *typ return nil } +// GeminiInteractionHelper relays native Gemini/Vertex interaction payloads +// without converting them into generateContent requests. +func GeminiInteractionHelper(c *gin.Context, info *relaycommon.RelayInfo, request *dto.GeminiInteractionRequest) (newAPIError *types.NewAPIError) { + if !strings.HasPrefix(info.OriginModelName, "lyria-3-") { + return types.NewErrorWithStatusCode(fmt.Errorf("unsupported gemini interaction model: %s", info.OriginModelName), types.ErrorCodeInvalidRequest, http.StatusBadRequest, types.ErrOptionWithSkipRetry()) + } + + err := helper.ModelMappedHelper(c, info, request) + if err != nil { + return types.NewError(err, types.ErrorCodeChannelModelMappedError, types.ErrOptionWithSkipRetry()) + } + + adaptor := GetAdaptor(info.ApiType) + if adaptor == nil { + return types.NewError(fmt.Errorf("invalid api type: %d", info.ApiType), types.ErrorCodeInvalidApiType, types.ErrOptionWithSkipRetry()) + } + + adaptor.Init(info) + + jsonData, err := common.Marshal(request) + if err != nil { + return types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) + } + + if len(info.ParamOverride) > 0 { + jsonData, err = relaycommon.ApplyParamOverrideWithRelayInfo(jsonData, info) + if err != nil { + return newAPIErrorFromParamOverride(err) + } + } + + logger.LogDebug(c, "Gemini interaction request body: "+string(jsonData)) + + resp, err := adaptor.DoRequest(c, info, bytes.NewReader(jsonData)) + if err != nil { + logger.LogError(c, "Do gemini interaction request failed: "+err.Error()) + return types.NewOpenAIError(err, types.ErrorCodeDoRequestFailed, http.StatusInternalServerError) + } + + statusCodeMappingStr := c.GetString("status_code_mapping") + httpResp, ok := resp.(*http.Response) + if !ok || httpResp == nil { + return types.NewErrorWithStatusCode(fmt.Errorf("invalid gemini interaction response type: %T", resp), types.ErrorCodeDoRequestFailed, http.StatusInternalServerError) + } + if httpResp.StatusCode != http.StatusOK { + newAPIError = service.RelayErrorHandler(c.Request.Context(), httpResp, false) + service.ResetStatusCode(newAPIError, statusCodeMappingStr) + return newAPIError + } + + usage, openaiErr := adaptor.DoResponse(c, httpResp, info) + if openaiErr != nil { + service.ResetStatusCode(openaiErr, statusCodeMappingStr) + return openaiErr + } + + service.PostTextConsumeQuota(c, info, usage.(*dto.Usage), nil) + return nil +} + func GeminiEmbeddingHandler(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *types.NewAPIError) { info.InitChannelMeta(c) diff --git a/relay/helper/valid_request.go b/relay/helper/valid_request.go index 2581b2812c94..bebc53f01af0 100644 --- a/relay/helper/valid_request.go +++ b/relay/helper/valid_request.go @@ -27,6 +27,8 @@ func GetAndValidateRequest(c *gin.Context, format types.RelayFormat) (request dt request, err = GetAndValidateGeminiEmbeddingRequest(c) } else if strings.Contains(c.Request.URL.Path, ":batchEmbedContents") { request, err = GetAndValidateGeminiBatchEmbeddingRequest(c) + } else if strings.Contains(c.Request.URL.Path, ":interactions") { + request, err = GetAndValidateGeminiInteractionRequest(c) } else { request, err = GetAndValidateGeminiRequest(c) } @@ -321,6 +323,28 @@ func GetAndValidateGeminiRequest(c *gin.Context) (*dto.GeminiChatRequest, error) return request, nil } +// GetAndValidateGeminiInteractionRequest validates native Vertex interaction +// payloads used by Lyria 3 on the Gemini relay path. +func GetAndValidateGeminiInteractionRequest(c *gin.Context) (*dto.GeminiInteractionRequest, error) { + request := &dto.GeminiInteractionRequest{} + err := common.UnmarshalBodyReusable(c, request) + if err != nil { + return nil, err + } + if request.Payload == nil { + return nil, errors.New("request body is required") + } + input, ok := request.Payload["input"] + if !ok { + return nil, errors.New("input is required") + } + inputItems, ok := input.([]any) + if !ok || len(inputItems) == 0 { + return nil, errors.New("input must be a non-empty array") + } + return request, nil +} + func GetAndValidateGeminiEmbeddingRequest(c *gin.Context) (*dto.GeminiEmbeddingRequest, error) { request := &dto.GeminiEmbeddingRequest{} err := common.UnmarshalBodyReusable(c, request) diff --git a/setting/ratio_setting/model_ratio.go b/setting/ratio_setting/model_ratio.go index 80702ee42ad2..e86ffeaf86c3 100644 --- a/setting/ratio_setting/model_ratio.go +++ b/setting/ratio_setting/model_ratio.go @@ -279,6 +279,8 @@ var defaultModelRatio = map[string]float64{ var defaultModelPrice = map[string]float64{ "suno_music": 0.1, "suno_lyrics": 0.01, + "lyria-3-pro-preview": 0.08, + "lyria-3-clip-preview": 0.04, "dall-e-3": 0.04, "imagen-3.0-generate-002": 0.03, "black-forest-labs/flux-1.1-pro": 0.04,