Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions dto/openai_image.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@ type ImageRequest struct {
OutputCompression json.RawMessage `json:"output_compression,omitempty"`
PartialImages json.RawMessage `json:"partial_images,omitempty"`
// Stream bool `json:"stream,omitempty"`
Watermark *bool `json:"watermark,omitempty"`
Image json.RawMessage `json:"image,omitempty"`
Watermark *bool `json:"watermark,omitempty"`
// zhipu 4v
WatermarkEnabled json.RawMessage `json:"watermark_enabled,omitempty"`
UserId json.RawMessage `json:"user_id,omitempty"`
Image json.RawMessage `json:"image,omitempty"`
// 用匿名参数接收额外参数
Extra map[string]json.RawMessage `json:"-"`
}
Expand Down
8 changes: 6 additions & 2 deletions relay/channel/zhipu_4v/adaptor.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInf
}

func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (any, error) {
//TODO implement me
return nil, errors.New("not implemented")
return request, nil
}

func (a *Adaptor) Init(info *relaycommon.RelayInfo) {
Expand All @@ -63,6 +62,8 @@ func (a *Adaptor) GetRequestURL(info *relaycommon.RelayInfo) (string, error) {
return fmt.Sprintf("%s/embeddings", specialPlan.OpenAIBaseURL), nil
}
return fmt.Sprintf("%s/api/paas/v4/embeddings", baseURL), nil
case relayconstant.RelayModeImagesGenerations:
return fmt.Sprintf("%s/api/paas/v4/images/generations", baseURL), nil
default:
if hasSpecialPlan && specialPlan.OpenAIBaseURL != "" {
return fmt.Sprintf("%s/chat/completions", specialPlan.OpenAIBaseURL), nil
Expand Down Expand Up @@ -114,6 +115,9 @@ func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycom
return claude.ClaudeHandler(c, resp, info, claude.RequestModeMessage)
}
default:
if info.RelayMode == relayconstant.RelayModeImagesGenerations {
return zhipu4vImageHandler(c, resp, info)
}
adaptor := openai.Adaptor{}
return adaptor.DoResponse(c, resp, info)
}
Expand Down
127 changes: 127 additions & 0 deletions relay/channel/zhipu_4v/image.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
package zhipu_4v

import (
"io"
"net/http"

"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/dto"
"github.com/QuantumNous/new-api/logger"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/service"
"github.com/QuantumNous/new-api/types"

"github.com/gin-gonic/gin"
)

type zhipuImageRequest struct {
Model string `json:"model"`
Prompt string `json:"prompt"`
Quality string `json:"quality,omitempty"`
Size string `json:"size,omitempty"`
WatermarkEnabled *bool `json:"watermark_enabled,omitempty"`
UserID string `json:"user_id,omitempty"`
}

type zhipuImageResponse struct {
Created *int64 `json:"created,omitempty"`
Data []zhipuImageData `json:"data,omitempty"`
ContentFilter any `json:"content_filter,omitempty"`
Usage *dto.Usage `json:"usage,omitempty"`
Error *zhipuImageError `json:"error,omitempty"`
RequestID string `json:"request_id,omitempty"`
ExtendParam map[string]string `json:"extendParam,omitempty"`
}

type zhipuImageError struct {
Code string `json:"code"`
Message string `json:"message"`
}

type zhipuImageData struct {
Url string `json:"url,omitempty"`
ImageUrl string `json:"image_url,omitempty"`
B64Json string `json:"b64_json,omitempty"`
B64Image string `json:"b64_image,omitempty"`
}

type openAIImagePayload struct {
Created int64 `json:"created"`
Data []openAIImageData `json:"data"`
}

type openAIImageData struct {
B64Json string `json:"b64_json"`
}

func zhipu4vImageHandler(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (*dto.Usage, *types.NewAPIError) {
responseBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeReadResponseBodyFailed, http.StatusInternalServerError)
}
service.CloseResponseBodyGracefully(resp)

var zhipuResp zhipuImageResponse
if err := common.Unmarshal(responseBody, &zhipuResp); err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}

if zhipuResp.Error != nil && zhipuResp.Error.Message != "" {
return nil, types.WithOpenAIError(types.OpenAIError{
Message: zhipuResp.Error.Message,
Type: "zhipu_image_error",
Code: zhipuResp.Error.Code,
}, resp.StatusCode)
}

payload := openAIImagePayload{}
if zhipuResp.Created != nil && *zhipuResp.Created != 0 {
payload.Created = *zhipuResp.Created
} else {
payload.Created = info.StartTime.Unix()
}
for _, data := range zhipuResp.Data {
url := data.Url
if url == "" {
url = data.ImageUrl
}
if url == "" {
logger.LogWarn(c, "zhipu_image_missing_url")
continue
}

var b64 string
switch {
case data.B64Json != "":
b64 = data.B64Json
case data.B64Image != "":
b64 = data.B64Image
default:
_, downloaded, err := service.GetImageFromUrl(url)
if err != nil {
logger.LogError(c, "zhipu_image_get_b64_failed: "+err.Error())
continue
}
b64 = downloaded
}

if b64 == "" {
logger.LogWarn(c, "zhipu_image_empty_b64")
continue
}

imageData := openAIImageData{
B64Json: b64,
}
payload.Data = append(payload.Data, imageData)
}
Comment on lines +83 to +117

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Images with only base64 but no URL are currently dropped

In the data loop, you require a non-empty URL before checking B64Json / B64Image:

url := data.Url
if url == "" {
    url = data.ImageUrl
}
if url == "" {
    logger.LogWarn(c, "zhipu_image_missing_url")
    continue
}

var b64 string
switch {
case data.B64Json != "":
    b64 = data.B64Json
case data.B64Image != "":
    b64 = data.B64Image
// ...

If zhipu returns entries that only contain base64 fields (no URL), those images will be skipped even though you already have usable data.

Reordering the logic so you prefer existing base64 fields and only fall back to fetching via URL when both are empty would avoid this data loss. For example:

-	for _, data := range zhipuResp.Data {
-		url := data.Url
-		if url == "" {
-			url = data.ImageUrl
-		}
-		if url == "" {
-			logger.LogWarn(c, "zhipu_image_missing_url")
-			continue
-		}
-
-		var b64 string
-		switch {
-		case data.B64Json != "":
-			b64 = data.B64Json
-		case data.B64Image != "":
-			b64 = data.B64Image
-		default:
-			_, downloaded, err := service.GetImageFromUrl(url)
-			if err != nil {
-				logger.LogError(c, "zhipu_image_get_b64_failed: "+err.Error())
-				continue
-			}
-			b64 = downloaded
-		}
+	for _, data := range zhipuResp.Data {
+		var b64 string
+		switch {
+		case data.B64Json != "":
+			b64 = data.B64Json
+		case data.B64Image != "":
+			b64 = data.B64Image
+		default:
+			url := data.Url
+			if url == "" {
+				url = data.ImageUrl
+			}
+			if url == "" {
+				logger.LogWarn(c, "zhipu_image_missing_url")
+				continue
+			}
+
+			_, downloaded, err := service.GetImageFromUrl(url)
+			if err != nil {
+				logger.LogError(c, "zhipu_image_get_b64_failed: "+err.Error())
+				continue
+			}
+			b64 = downloaded
+		}

The rest of the block (b64 == "" check and appending to payload.Data) can remain unchanged.

🤖 Prompt for AI Agents
In relay/channel/zhipu_4v/image.go around lines 84 to 118, the loop currently
rejects entries with no URL before checking B64Json/B64Image, causing images
that only contain base64 to be dropped; change the logic to first check
data.B64Json and data.B64Image and use those if present, and only when both
base64 fields are empty then resolve url (data.Url or data.ImageUrl) and call
service.GetImageFromUrl; log the "missing url" or "get b64 failed"
warnings/errors only when neither base64 is present and fetching by URL fails or
is absent, then proceed to the existing b64=="" check and append to
payload.Data.


jsonResp, err := common.Marshal(payload)
if err != nil {
return nil, types.NewError(err, types.ErrorCodeBadResponseBody)
}

service.IOCopyBytesGracefully(c, resp, jsonResp)

return &dto.Usage{}, nil
}