Skip to content

Gemini Veo3.1[AI Studio]增加图生视频支持 #2315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
creamlike1024 merged 2 commits into from
Nov 27, 2025
Merged

Gemini Veo3.1[AI Studio]增加图生视频支持 #2315

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4a367ed
feat: gemini video veo3.1 add i2v
feitianbubu Nov 26, 2025
f7d6c36
feat: gemini video veo3.1 add task fail check
feitianbubu Nov 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 62 additions & 28 deletions relay/channel/task/gemini/adaptor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,29 +24,31 @@ import (
"github.com/pkg/errors"
)

// ============================
// Request / Response structures
// ============================

// GeminiVideoGenerationConfig represents the video generation configuration
// VideoGenerationConfig represents the video generation configuration
// Based on: https://ai.google.dev/gemini-api/docs/video
type GeminiVideoGenerationConfig struct {
type VideoGenerationConfig struct {
AspectRatio string `json:"aspectRatio,omitempty"` // "16:9" or "9:16"
DurationSeconds float64 `json:"durationSeconds,omitempty"` // 4, 6, or 8 (as number)
NegativePrompt string `json:"negativePrompt,omitempty"` // unwanted elements
PersonGeneration string `json:"personGeneration,omitempty"` // "allow_all" for text-to-video, "allow_adult" for image-to-video
Resolution string `json:"resolution,omitempty"` // video resolution
}

// GeminiVideoRequest represents a single video generation instance
type GeminiVideoRequest struct {
Prompt string `json:"prompt"`
type Image struct {
BytesBase64Encoded string `json:"bytesBase64Encoded,omitempty"`
MimeType string `json:"mimeType,omitempty"`
}

// GeminiVideoPayload represents the complete video generation request payload
type GeminiVideoPayload struct {
Instances []GeminiVideoRequest `json:"instances"`
Parameters GeminiVideoGenerationConfig `json:"parameters,omitempty"`
type VideoRequest struct {
Prompt string `json:"prompt"`
Image *Image `json:"image,omitempty"`
LastFrame *Image `json:"lastFrame,omitempty"`
}

// VideoPayload represents the complete video generation request payload
type VideoPayload struct {
Instances []VideoRequest `json:"instances"`
Parameters VideoGenerationConfig `json:"parameters,omitempty"`
}

type submitResponse struct {
Expand Down Expand Up @@ -75,6 +77,8 @@ type operationResponse struct {
URI string `json:"uri"`
} `json:"video"`
} `json:"generatedSamples"`
RaiMediaFilteredCount int `json:"raiMediaFilteredCount"`
RaiMediaFilteredReasons []string `json:"raiMediaFilteredReasons"`
} `json:"generateVideoResponse"`
} `json:"response"`
Error struct {
Expand All @@ -100,8 +104,7 @@ func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {

// ValidateRequestAndSetAction parses body, validates fields and sets default action.
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *dto.TaskError) {
// Use the standard validation method for TaskSubmitReq
return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionTextGenerate)
return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionGenerate)
}

// BuildRequestURL constructs the upstream URL.
Expand Down Expand Up @@ -137,13 +140,21 @@ func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayIn
}

// Create structured video generation request
body := GeminiVideoPayload{
Instances: []GeminiVideoRequest{
body := VideoPayload{
Instances: []VideoRequest{
{Prompt: req.Prompt},
},
Parameters: GeminiVideoGenerationConfig{},
Parameters: VideoGenerationConfig{},
}

if len(req.Images) > 0 {
body.Instances[0].Image = a.convertImage(req.Images[0])
}
if len(req.Images) > 1 {
body.Instances[0].LastFrame = a.convertImage(req.Images[1])
}
Comment on lines +150 to 155

@coderabbitai coderabbitai Bot Nov 26, 2025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Silent failure when image fetch fails.

convertImage silently returns an Image with the original URL string as BytesBase64Encoded when service.GetImageFromUrl fails. This will likely cause the upstream API to reject the request with a confusing error.

Consider returning an error from convertImage or at least logging the failure:

-func (a *TaskAdaptor) convertImage(imageStr string) *Image {
+func (a *TaskAdaptor) convertImage(imageStr string) (*Image, error) {
 	if strings.TrimSpace(imageStr) == "" {
-		return nil
+		return nil, nil
 	}
-	img := &Image{
-		MimeType:           "image/png",
-		BytesBase64Encoded: imageStr,
-	}
 	if strings.HasPrefix(imageStr, "data:image/") {
 		parts := strings.Split(imageStr, ";base64,")
 		if len(parts) == 2 {
-			img.MimeType = strings.TrimPrefix(parts[0], "data:")
-			img.BytesBase64Encoded = parts[1]
+			return &Image{
+				MimeType:           strings.TrimPrefix(parts[0], "data:"),
+				BytesBase64Encoded: parts[1],
+			}, nil
 		}
+		return nil, fmt.Errorf("invalid data URL format")
 	} else if strings.HasPrefix(imageStr, "http") {
 		mimeType, data, err := service.GetImageFromUrl(imageStr)
-		if err == nil {
-			img.MimeType = mimeType
-			img.BytesBase64Encoded = data
+		if err != nil {
+			return nil, fmt.Errorf("fetch image failed: %w", err)
 		}
+		return &Image{MimeType: mimeType, BytesBase64Encoded: data}, nil
 	}
-	return img
+	// Assume raw base64
+	return &Image{MimeType: "image/png", BytesBase64Encoded: imageStr}, nil
 }

Then update BuildRequestBody to handle the error:

 	if len(req.Images) > 0 {
-		body.Instances[0].Image = a.convertImage(req.Images[0])
+		img, err := a.convertImage(req.Images[0])
+		if err != nil {
+			return nil, errors.Wrap(err, "convert image failed")
+		}
+		body.Instances[0].Image = img
 	}

Committable suggestion skipped: line range outside the PR's diff.

🤖 Prompt for AI Agents 
In relay/channel/task/gemini/adaptor.go around lines 150-155, convertImage
currently swallows failures from service.GetImageFromUrl and returns an Image
with the original URL in BytesBase64Encoded causing upstream rejections; change
convertImage to return (Image, error) (or at minimum log the specific fetch
error), propagate and handle that error in BuildRequestBody so you either return
the error to the caller or skip/omit the image field when fetching fails, and
update the two call sites in BuildRequestBody to check the error and act
accordingly (return a wrapped error with context or skip setting
body.Instances[0].Image/LastFrame).


// Parse metadata for additional configuration
metadata := req.Metadata
medaBytes, err := json.Marshal(metadata)
if err != nil {
Expand Down Expand Up @@ -247,20 +258,19 @@ func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, e
return ti, nil
}

ti.Status = model.TaskStatusSuccess
ti.Progress = "100%"

taskID := encodeLocalTaskID(op.Name)
ti.TaskID = taskID
ti.Url = fmt.Sprintf("%s/v1/videos/%s/content", system_setting.ServerAddress, taskID)

// Extract URL from generateVideoResponse if available
if len(op.Response.GenerateVideoResponse.GeneratedSamples) > 0 {
if len(op.Response.GenerateVideoResponse.GeneratedSamples) == 0 {
ti.Status = model.TaskStatusFailure
ti.Reason = fmt.Sprintf("no generated video url found: %s", strings.Join(op.Response.GenerateVideoResponse.RaiMediaFilteredReasons, "; "))
} else {
if uri := op.Response.GenerateVideoResponse.GeneratedSamples[0].Video.URI; uri != "" {
ti.RemoteUrl = uri
}
ti.Status = model.TaskStatusSuccess
}

ti.Progress = "100%"
taskID := encodeLocalTaskID(op.Name)
ti.TaskID = taskID
ti.Url = fmt.Sprintf("%s/v1/videos/%s/content", system_setting.ServerAddress, taskID)
return ti, nil
Comment on lines +261 to 274

@coderabbitai coderabbitai Bot Nov 26, 2025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

URL is set even when generation fails.

When no GeneratedSamples exist (RAI filtered), the status is set to FAILURE but ti.Url is still populated on line 273. This URL would point to non-existent content, potentially confusing clients.

Consider only setting the URL on success:

 	if len(op.Response.GenerateVideoResponse.GeneratedSamples) == 0 {
 		ti.Status = model.TaskStatusFailure
 		ti.Reason = fmt.Sprintf("no generated video url found: %s", strings.Join(op.Response.GenerateVideoResponse.RaiMediaFilteredReasons, "; "))
+		ti.Progress = "100%"
 	} else {
 		if uri := op.Response.GenerateVideoResponse.GeneratedSamples[0].Video.URI; uri != "" {
 			ti.RemoteUrl = uri
 		}
 		ti.Status = model.TaskStatusSuccess
+		ti.Progress = "100%"
+		taskID := encodeLocalTaskID(op.Name)
+		ti.TaskID = taskID
+		ti.Url = fmt.Sprintf("%s/v1/videos/%s/content", system_setting.ServerAddress, taskID)
 	}
-	ti.Progress = "100%"
-	taskID := encodeLocalTaskID(op.Name)
-	ti.TaskID = taskID
-	ti.Url = fmt.Sprintf("%s/v1/videos/%s/content", system_setting.ServerAddress, taskID)
 	return ti, nil
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
if len(op.Response.GenerateVideoResponse.GeneratedSamples) == 0 {
ti.Status = model.TaskStatusFailure
ti.Reason = fmt.Sprintf("no generated video url found: %s", strings.Join(op.Response.GenerateVideoResponse.RaiMediaFilteredReasons, "; "))
} else {
if uri := op.Response.GenerateVideoResponse.GeneratedSamples[0].Video.URI; uri != "" {
ti.RemoteUrl = uri
}
ti.Status = model.TaskStatusSuccess
}
ti.Progress = "100%"
taskID := encodeLocalTaskID(op.Name)
ti.TaskID = taskID
ti.Url = fmt.Sprintf("%s/v1/videos/%s/content", system_setting.ServerAddress, taskID)
return ti, nil
if len(op.Response.GenerateVideoResponse.GeneratedSamples) == 0 {
ti.Status = model.TaskStatusFailure
ti.Reason = fmt.Sprintf("no generated video url found: %s", strings.Join(op.Response.GenerateVideoResponse.RaiMediaFilteredReasons, "; "))
ti.Progress = "100%"
} else {
if uri := op.Response.GenerateVideoResponse.GeneratedSamples[0].Video.URI; uri != "" {
ti.RemoteUrl = uri
}
ti.Status = model.TaskStatusSuccess
ti.Progress = "100%"
taskID := encodeLocalTaskID(op.Name)
ti.TaskID = taskID
ti.Url = fmt.Sprintf("%s/v1/videos/%s/content", system_setting.ServerAddress, taskID)
}
return ti, nil
🤖 Prompt for AI Agents 
In relay/channel/task/gemini/adaptor.go around lines 261 to 274, the code sets
ti.Url unconditionally even when generation failed; move the ti.Url assignment
so it is only set when the task status is success (i.e., inside the branch where
GeneratedSamples exist and after setting ti.TaskID), and ensure that on failure
ti.Url is left empty (or explicitly cleared) so clients won't receive a pointer
to non-existent content.

}

Expand Down Expand Up @@ -289,6 +299,30 @@ func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) {
return common.Marshal(video)
}

func (a *TaskAdaptor) convertImage(imageStr string) *Image {
if strings.TrimSpace(imageStr) == "" {
return nil
}
img := &Image{
MimeType: "image/png",
BytesBase64Encoded: imageStr,
}
if strings.HasPrefix(imageStr, "data:image/") {
parts := strings.Split(imageStr, ";base64,")
if len(parts) == 2 {
img.MimeType = strings.TrimPrefix(parts[0], "data:")
img.BytesBase64Encoded = parts[1]
}
} else if strings.HasPrefix(imageStr, "http") {
mimeType, data, err := service.GetImageFromUrl(imageStr)
if err == nil {
img.MimeType = mimeType
img.BytesBase64Encoded = data
}
}
return img
}

// ============================
// helpers
// ============================
Expand Down