diff --git a/core/bifrost.go b/core/bifrost.go index e2cdbc51eda..f2e8afcdf5c 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -1905,11 +1905,13 @@ func (bifrost *Bifrost) VideoGenerationRequest(ctx *schemas.BifrostContext, }, } } - if req.Input == nil || req.Input.Prompt == "" { + // Operations driven by an input asset (video upscale, image-to-3D) carry no prompt; the + // provider rejects a combination its model cannot serve. + if req.Input == nil || (req.Input.Prompt == "" && req.Input.InputReference == nil && req.Input.VideoURI == nil) { return nil, &schemas.BifrostError{ IsBifrostError: false, Error: &schemas.ErrorField{ - Message: "prompt not provided for video generation request", + Message: "prompt, input_reference or video_uri not provided for video generation request", }, ExtraFields: schemas.BifrostErrorExtraFields{ RequestType: schemas.VideoGenerationRequest, diff --git a/core/providers/gemini/images.go b/core/providers/gemini/images.go index 0663e411c02..f71552ee9ad 100644 --- a/core/providers/gemini/images.go +++ b/core/providers/gemini/images.go @@ -846,6 +846,9 @@ func ToGeminiImageEditRequest(bifrostReq *schemas.BifrostImageEditRequest) *Gemi } for _, image := range bifrostReq.Input.Images { + if len(image.Image) == 0 { + continue + } // Detect MIME type from image bytes mimeType := http.DetectContentType(image.Image) // Fallback to PNG if detection fails diff --git a/core/providers/gemini/videos.go b/core/providers/gemini/videos.go index 54f0472550a..5d8be636923 100644 --- a/core/providers/gemini/videos.go +++ b/core/providers/gemini/videos.go @@ -277,9 +277,9 @@ func ToGeminiVideoGenerationRequest(bifrostReq *schemas.BifrostVideoGenerationRe instance.Image = image } - if bifrostReq.Params != nil && bifrostReq.Params.VideoURI != nil { + if bifrostReq.Input != nil && bifrostReq.Input.VideoURI != nil { instance.Video = &VideoGenerationVideoInput{ - URI: bifrostReq.Params.VideoURI, + URI: bifrostReq.Input.VideoURI, } } @@ -586,8 +586,7 @@ func (request *GeminiVideoGenerationRequest) ToBifrostVideoGenerationRequest(ctx // Handle video URI if instance.Video != nil && instance.Video.URI != nil { - ensureParams() - bifrostReq.Params.VideoURI = instance.Video.URI + bifrostReq.Input.VideoURI = instance.Video.URI } // Handle last frame diff --git a/core/providers/openai/types.go b/core/providers/openai/types.go index 8c722f9519f..f072009e79f 100644 --- a/core/providers/openai/types.go +++ b/core/providers/openai/types.go @@ -1134,8 +1134,9 @@ var ValidOpenAIVideoSizes = map[string]bool{ // OpenAIVideoGenerationRequest is the request body for OpenAI video generation. type OpenAIVideoGenerationRequest struct { - Prompt string `json:"prompt"` // Text prompt that describes the video to generate (max 32000, min 1) - InputReference []byte `json:"input_reference,omitempty"` // Optional image reference file that guides generation + Prompt string `json:"prompt"` // Text prompt that describes the video to generate (max 32000, min 1) + InputReference []byte `json:"input_reference,omitempty"` // Optional image reference file that guides generation + VideoURI *string `json:"video_uri,omitempty"` // Optional source video for video-to-video Model string `json:"model"` // Video generation model (defaults to sora-2) diff --git a/core/providers/openai/videos.go b/core/providers/openai/videos.go index 1b794b8ad3e..fec1e4d17dc 100644 --- a/core/providers/openai/videos.go +++ b/core/providers/openai/videos.go @@ -103,7 +103,8 @@ func (req *OpenAIVideoGenerationRequest) ToBifrostVideoGenerationRequest(ctx *sc provider, model := schemas.ParseModelString(req.Model, "") input := &schemas.VideoGenerationInput{ - Prompt: req.Prompt, + Prompt: req.Prompt, + VideoURI: req.VideoURI, } if req.InputReference != nil { input.InputReference = schemas.Ptr(providerUtils.FileBytesToBase64DataURL(req.InputReference)) diff --git a/core/providers/runware/images.go b/core/providers/runware/images.go index 4b6253e6e25..587ad5a6c4f 100644 --- a/core/providers/runware/images.go +++ b/core/providers/runware/images.go @@ -2,6 +2,7 @@ package runware import ( "fmt" + "strings" "github.com/google/uuid" providerUtils "github.com/maximhq/bifrost/core/providers/utils" @@ -16,14 +17,22 @@ func ToRunwareImageGenerationRequest(bifrostReq *schemas.BifrostImageGenerationR return nil, fmt.Errorf("input is required") } + // Text-to-SVG runs as its own task type on this endpoint; everything else is imageInference. + taskType := taskTypeImageInference + if bifrostReq.Params != nil && bifrostReq.Params.Type != nil && + strings.EqualFold(strings.TrimSpace(*bifrostReq.Params.Type), "vectorize") { + taskType = taskTypeVectorize + } + width, height := defaultRunwareWidth, defaultRunwareHeight request := &RunwareInferenceRequest{ - TaskType: taskTypeImageInference, + TaskType: taskType, TaskUUID: uuid.New().String(), Model: bifrostReq.Model, PositivePrompt: &bifrostReq.Input.Prompt, Width: &width, Height: &height, + IncludeCost: new(true), } if bifrostReq.Params != nil { @@ -59,10 +68,14 @@ func ToRunwareImageEditRequest(bifrostReq *schemas.BifrostImageEditRequest) (*Ru if bifrostReq.Input == nil { return nil, fmt.Errorf("input is required") } - if len(bifrostReq.Input.Images) == 0 || len(bifrostReq.Input.Images[0].Image) == 0 { + if len(bifrostReq.Input.Images) == 0 || runwareImageInput(bifrostReq.Input.Images[0]) == "" { return nil, fmt.Errorf("at least one input image is required") } + if taskType := runwareImageEditTaskType(bifrostReq.Params); taskType != "" { + return toRunwareImageToolRequest(taskType, bifrostReq) + } + width, height := defaultRunwareWidth, defaultRunwareHeight request := &RunwareInferenceRequest{ TaskType: taskTypeImageInference, @@ -71,10 +84,11 @@ func ToRunwareImageEditRequest(bifrostReq *schemas.BifrostImageEditRequest) (*Ru PositivePrompt: &bifrostReq.Input.Prompt, Width: &width, Height: &height, + IncludeCost: new(true), } // Seed image: the base image being edited (raw bytes -> base64 data URI). - seedImage := providerUtils.FileBytesToBase64DataURL(bifrostReq.Input.Images[0].Image) + seedImage := runwareImageInput(bifrostReq.Input.Images[0]) request.SeedImage = &seedImage if bifrostReq.Params != nil { @@ -102,6 +116,95 @@ func ToRunwareImageEditRequest(bifrostReq *schemas.BifrostImageEditRequest) (*Ru return request, nil } +// runwareImageInput resolves an input image to the reference Runware expects. A caller-supplied +// URL passes through untouched — Runware accepts UUIDs and URLs natively, so forwarding it avoids +// round-tripping the asset through the gateway as base64 — while raw bytes become a data URI. +func runwareImageInput(img schemas.ImageInput) string { + if img.URL != "" { + return img.URL + } + if len(img.Image) == 0 { + return "" + } + return providerUtils.FileBytesToBase64DataURL(img.Image) +} + +// runwareImageEditTaskType maps the neutral edit type onto a Runware tool task type. An empty +// result means the edit runs as a regular imageInference task (image-to-image, inpainting, +// outpainting). +func runwareImageEditTaskType(params *schemas.ImageEditParameters) string { + if params == nil || params.Type == nil { + return "" + } + switch strings.ReplaceAll(strings.ToLower(strings.TrimSpace(*params.Type)), "-", "_") { + case "upscale": + return taskTypeUpscale + case "background_removal", "remove_background", "remove_bg": + return taskTypeRemoveBackground + case "mask", "segmentation": + return taskTypeImageMasking + case "vectorize": + return taskTypeVectorize + } + return "" +} + +// runwareResultAssets resolves a task result's output family. Runware names an artifact after what +// the task produces, so masking returns maskImage* and ControlNet preprocessing returns +// guideImage* rather than reusing image*; reading only image* would silently drop the output. +func runwareResultAssets(result *RunwareResult) (id string, url string, base64Data string, dataURI string) { + switch { + case result.ImageURL != "", result.ImageBase64Data != "", result.ImageDataURI != "": + return result.ImageUUID, result.ImageURL, result.ImageBase64Data, result.ImageDataURI + case result.MaskImageURL != "", result.MaskImageBase64Data != "", result.MaskImageDataURI != "": + return result.MaskImageUUID, result.MaskImageURL, result.MaskImageBase64Data, result.MaskImageDataURI + case result.GuideImageURL != "", result.GuideImageBase64Data != "", result.GuideImageDataURI != "": + return result.GuideImageUUID, result.GuideImageURL, result.GuideImageBase64Data, result.GuideImageDataURI + } + return result.ImageUUID, "", "", "" +} + +// toRunwareImageToolRequest builds a Runware single-image tool task (upscale, removeBackground). +// These share one envelope: the image is nested under "inputs", model tuning goes in "settings" +// and "providerSettings", and none of the imageInference fields (prompt, dimensions, steps) apply, +// so they are left unset. Runware-native fields are read from extra params under their own names. +func toRunwareImageToolRequest(taskType string, bifrostReq *schemas.BifrostImageEditRequest) (*RunwareInferenceRequest, error) { + image := runwareImageInput(bifrostReq.Input.Images[0]) + request := &RunwareInferenceRequest{ + TaskType: taskType, + TaskUUID: uuid.New().String(), + Model: bifrostReq.Model, + Inputs: &RunwareInputs{Image: &image}, + IncludeCost: new(true), + } + + if bifrostReq.Params == nil { + return request, nil + } + params := bifrostReq.Params + + request.OutputType = runwareOutputType(params.ResponseFormat) + request.OutputFormat = runwareOutputFormat(params.OutputFormat) + request.OutputQuality = params.OutputCompression + request.ExtraParams = params.ExtraParams + + // Consume the fields promoted to typed properties so they are not also re-sent verbatim + // when extra-param passthrough is enabled. + if v, ok := runwareSettings(request.ExtraParams["settings"]); ok { + delete(request.ExtraParams, "settings") + request.Settings = v + } + if v, ok := runwareSettings(request.ExtraParams["providerSettings"]); ok { + delete(request.ExtraParams, "providerSettings") + request.ProviderSettings = v + } + + request.UpscaleFactor = params.UpscaleFactor + request.TargetMegapixels = params.TargetMegapixels + + return request, nil +} + // ToBifrostImageGenerationResponse converts a Runware response envelope to a Bifrost image response. func ToBifrostImageGenerationResponse(resp *RunwareResponse) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { if resp == nil { @@ -125,13 +228,23 @@ func ToBifrostImageGenerationResponse(resp *RunwareResponse) (*schemas.BifrostIm var totalCost float64 for i, img := range resp.Data { data := schemas.ImageData{Index: i} + id, url, base64Data, dataURI := runwareResultAssets(&img) + // Runware accepts these UUIDs as inputs, so surfacing them lets callers chain tasks + // (mask then inpaint, upscale then remove background) without re-uploading the asset. + data.ID = id switch { - case img.ImageURL != "": - data.URL = img.ImageURL - case img.ImageBase64Data != "": - data.B64JSON = img.ImageBase64Data - case img.ImageDataURI != "": - data.URL = img.ImageDataURI + case url != "": + data.URL = url + case base64Data != "": + data.B64JSON = base64Data + case dataURI != "": + data.URL = dataURI + } + // Masking models report the regions they located alongside the mask itself. + for _, d := range img.Detections { + data.Detections = append(data.Detections, schemas.ImageDetection{ + XMin: d.XMin, YMin: d.YMin, XMax: d.XMax, YMax: d.YMax, + }) } bifrostResp.Data = append(bifrostResp.Data, data) if img.Seed != nil { diff --git a/core/providers/runware/images_test.go b/core/providers/runware/images_test.go index 6f344f686eb..55a3b1daf2d 100644 --- a/core/providers/runware/images_test.go +++ b/core/providers/runware/images_test.go @@ -1,9 +1,280 @@ package runware import ( + "strings" "testing" + + schemas "github.com/maximhq/bifrost/core/schemas" ) +func upscaleEditRequest(extraParams map[string]interface{}) *schemas.BifrostImageEditRequest { + return &schemas.BifrostImageEditRequest{ + Model: "topazlabs:wonder@3.5", + Input: &schemas.ImageEditInput{Images: []schemas.ImageInput{{Image: []byte("fake-image-bytes")}}}, + Params: &schemas.ImageEditParameters{ + Type: new("upscale"), + ExtraParams: extraParams, + }, + } +} + +// type=upscale switches the edit path to the upscale task: the image moves under inputs, and the +// imageInference-only fields (prompt, dimensions, steps) are left unset since Runware rejects them. +func TestToRunwareImageEditRequest_Upscale(t *testing.T) { + out, err := ToRunwareImageEditRequest(upscaleEditRequest(nil)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.TaskType != taskTypeUpscale { + t.Fatalf("taskType = %q, want %q", out.TaskType, taskTypeUpscale) + } + if out.Inputs == nil || out.Inputs.Image == nil || !strings.HasPrefix(*out.Inputs.Image, "data:") { + t.Fatalf("expected inputs.image data URI, got %+v", out.Inputs) + } + if out.SeedImage != nil { + t.Fatalf("seedImage must stay unset on upscale, got %q", *out.SeedImage) + } + if out.PositivePrompt != nil || out.Width != nil || out.Height != nil || out.Steps != nil { + t.Fatalf("prompt/width/height/steps must stay unset on upscale, got %+v", out) + } + if out.IncludeCost == nil || !*out.IncludeCost { + t.Fatalf("expected includeCost=true so the task cost is reported") + } +} + +// Upscaler fields arrive as multipart form strings; they are coerced to typed properties and +// removed from ExtraParams so passthrough does not also emit them verbatim. +func TestToRunwareImageEditRequest_UpscaleExtraParams(t *testing.T) { + req := upscaleEditRequest(map[string]interface{}{ + "settings": `{"enhancementStrength":"high"}`, + "unrelated": "keep-me", + }) + req.Params.UpscaleFactor = new(4) + req.Params.TargetMegapixels = new(16) + + out, err := ToRunwareImageEditRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.UpscaleFactor == nil || *out.UpscaleFactor != 4 { + t.Fatalf("upscaleFactor = %v, want 4", out.UpscaleFactor) + } + if out.TargetMegapixels == nil || *out.TargetMegapixels != 16 { + t.Fatalf("targetMegapixels = %v, want 16", out.TargetMegapixels) + } + if out.Settings["enhancementStrength"] != "high" { + t.Fatalf("settings = %+v, want enhancementStrength=high", out.Settings) + } + for _, consumed := range []string{"settings"} { + if _, ok := out.ExtraParams[consumed]; ok { + t.Fatalf("%q should be consumed from ExtraParams, got %+v", consumed, out.ExtraParams) + } + } + if out.ExtraParams["unrelated"] != "keep-me" { + t.Fatalf("unrecognised extra params must pass through, got %+v", out.ExtraParams) + } +} + +// type=background_removal (and its aliases) selects the removeBackground task, which shares the +// tool envelope with upscale but takes none of the upscaler-specific fields. +func TestToRunwareImageEditRequest_RemoveBackground(t *testing.T) { + for _, alias := range []string{"background_removal", "remove_background", "remove_bg", "Remove-Background"} { + req := upscaleEditRequest(nil) + req.Model = "runware:109@1" + req.Params.Type = &alias + + out, err := ToRunwareImageEditRequest(req) + if err != nil { + t.Fatalf("%s: unexpected error: %v", alias, err) + } + if out.TaskType != taskTypeRemoveBackground { + t.Fatalf("%s: taskType = %q, want %q", alias, out.TaskType, taskTypeRemoveBackground) + } + if out.Inputs == nil || out.Inputs.Image == nil { + t.Fatalf("%s: expected inputs.image, got %+v", alias, out.Inputs) + } + if out.UpscaleFactor != nil || out.TargetMegapixels != nil { + t.Fatalf("%s: upscaler fields must stay unset, got %+v", alias, out) + } + if out.PositivePrompt != nil || out.Width != nil || out.Height != nil { + t.Fatalf("%s: prompt/dimensions must stay unset, got %+v", alias, out) + } + if out.IncludeCost == nil || !*out.IncludeCost { + t.Fatalf("%s: expected includeCost=true", alias) + } + } +} + +// removeBackground tuning lives in settings (runware models) or providerSettings (bria); both are +// consumed out of extra params so they reach the wire as nested objects. +func TestToRunwareImageEditRequest_RemoveBackgroundSettings(t *testing.T) { + req := upscaleEditRequest(map[string]interface{}{ + "settings": `{"returnOnlyMask":true,"rgba":[255,255,255,0]}`, + "providerSettings": map[string]interface{}{"bria": map[string]interface{}{"preserveAlpha": true}}, + "unrelated": "keep-me", + }) + req.Model = "bria:2@1" + req.Params.Type = new("background_removal") + + out, err := ToRunwareImageEditRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Settings["returnOnlyMask"] != true { + t.Fatalf("settings = %+v, want returnOnlyMask=true", out.Settings) + } + if out.ProviderSettings["bria"] == nil { + t.Fatalf("providerSettings = %+v, want a bria entry", out.ProviderSettings) + } + if out.ExtraParams["unrelated"] != "keep-me" { + t.Fatalf("unrecognised extra params must pass through, got %+v", out.ExtraParams) + } +} + +// A settings object supplied by a JSON caller is used as-is, without a string round-trip. +func TestToRunwareImageEditRequest_UpscaleSettingsObject(t *testing.T) { + out, err := ToRunwareImageEditRequest(upscaleEditRequest(map[string]interface{}{ + "settings": map[string]interface{}{"realism": true}, + })) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Settings["realism"] != true { + t.Fatalf("settings = %+v, want realism=true", out.Settings) + } +} + +// Edits without type=upscale keep using imageInference with a top-level seedImage. +func TestToRunwareImageEditRequest_NonUpscaleUnchanged(t *testing.T) { + out, err := ToRunwareImageEditRequest(&schemas.BifrostImageEditRequest{ + Model: "runware:101@1", + Input: &schemas.ImageEditInput{ + Images: []schemas.ImageInput{{Image: []byte("fake-image-bytes")}}, + Prompt: "make it night", + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.TaskType != taskTypeImageInference { + t.Fatalf("taskType = %q, want %q", out.TaskType, taskTypeImageInference) + } + if out.SeedImage == nil { + t.Fatalf("expected seedImage to remain top-level for imageInference") + } + if out.Inputs != nil { + t.Fatalf("inputs must stay unset for imageInference, got %+v", out.Inputs) + } +} + +// type=mask selects the imageMasking task; detector settings ride along in settings. +func TestToRunwareImageEditRequest_Mask(t *testing.T) { + for _, alias := range []string{"mask", "segmentation", "Mask"} { + req := upscaleEditRequest(map[string]interface{}{ + "settings": `{"confidence":0.7,"maskPadding":20,"maxDetections":4}`, + }) + req.Model = "runware:35@1" + req.Params.Type = &alias + + out, err := ToRunwareImageEditRequest(req) + if err != nil { + t.Fatalf("%s: unexpected error: %v", alias, err) + } + if out.TaskType != taskTypeImageMasking { + t.Fatalf("%s: taskType = %q, want %q", alias, out.TaskType, taskTypeImageMasking) + } + if out.Inputs == nil || out.Inputs.Image == nil { + t.Fatalf("%s: expected inputs.image, got %+v", alias, out.Inputs) + } + if out.Settings["confidence"] != 0.7 { + t.Fatalf("%s: settings = %+v, want confidence=0.7", alias, out.Settings) + } + } +} + +// Masking and ControlNet preprocessing return their artifact under maskImage*/guideImage*; reading +// only image* would emit an entry with no URL at all. Detections ride along on the same entry. +func TestToBifrostImageGenerationResponse_OutputFamilies(t *testing.T) { + t.Run("maskImage with detections", func(t *testing.T) { + out, err := ToBifrostImageGenerationResponse(&RunwareResponse{Data: []RunwareResult{{ + TaskUUID: "mask-1", + MaskImageURL: "https://x/mask.png", + Detections: []RunwareDetection{{XMin: 1, YMin: 2, XMax: 3, YMax: 4}}, + }}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Data[0].URL != "https://x/mask.png" { + t.Fatalf("mask URL not surfaced: %+v", out.Data[0]) + } + if len(out.Data[0].Detections) != 1 || out.Data[0].Detections[0].XMax != 3 { + t.Fatalf("detections not surfaced: %+v", out.Data[0].Detections) + } + }) + + t.Run("guideImage base64", func(t *testing.T) { + out, err := ToBifrostImageGenerationResponse(&RunwareResponse{Data: []RunwareResult{{ + TaskUUID: "guide-1", + GuideImageBase64Data: "Zm9v", + }}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Data[0].B64JSON != "Zm9v" { + t.Fatalf("guide image not surfaced: %+v", out.Data[0]) + } + }) + + t.Run("image family still wins", func(t *testing.T) { + out, err := ToBifrostImageGenerationResponse(&RunwareResponse{Data: []RunwareResult{{ + TaskUUID: "img-1", ImageURL: "https://x/a.png", MaskImageURL: "https://x/mask.png", + }}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Data[0].URL != "https://x/a.png" { + t.Fatalf("image family must take precedence, got %+v", out.Data[0]) + } + if len(out.Data[0].Detections) != 0 { + t.Fatalf("expected no detections, got %+v", out.Data[0].Detections) + } + }) +} + +// Runware only returns a per-task cost when the request opts in, so every generated task sets +// includeCost; without it the response carries no cost and the task is billed as $0. +func TestToRunwareRequests_AlwaysIncludeCost(t *testing.T) { + gen, err := ToRunwareImageGenerationRequest(&schemas.BifrostImageGenerationRequest{ + Model: "runware:101@1", + Input: &schemas.ImageGenerationInput{Prompt: "a teapot"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + edit, err := ToRunwareImageEditRequest(&schemas.BifrostImageEditRequest{ + Model: "runware:101@1", + Input: &schemas.ImageEditInput{ + Images: []schemas.ImageInput{{Image: []byte("fake-image-bytes")}}, + Prompt: "make it night", + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + upscale, err := ToRunwareImageEditRequest(upscaleEditRequest(nil)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for name, req := range map[string]*RunwareInferenceRequest{ + "generation": gen, "edit": edit, "upscale": upscale, + } { + if req.IncludeCost == nil || !*req.IncludeCost { + t.Fatalf("%s: expected includeCost=true, got %v", name, req.IncludeCost) + } + } +} + // Runware reports per-task cost (when includeCost is set); it is summed across results and surfaced // as the provider-reported image cost so pricing uses it verbatim. func TestToBifrostImageGenerationResponse_Cost(t *testing.T) { @@ -37,3 +308,107 @@ func TestToBifrostImageGenerationResponse_NoCost(t *testing.T) { t.Fatalf("expected nil Usage when cost absent, got %+v", out.Usage) } } + +// type=vectorize drives the raster-to-SVG task on the edit path, and the prompt-to-SVG task on the +// generation path, where the models take a prompt and no input image. +func TestToRunwareVectorize(t *testing.T) { + edit := upscaleEditRequest(nil) + edit.Model = "recraft:1@1" + edit.Params.Type = new("vectorize") + editOut, err := ToRunwareImageEditRequest(edit) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if editOut.TaskType != taskTypeVectorize { + t.Fatalf("edit taskType = %q, want %q", editOut.TaskType, taskTypeVectorize) + } + if editOut.Inputs == nil || editOut.Inputs.Image == nil { + t.Fatalf("edit expects inputs.image, got %+v", editOut.Inputs) + } + + gen, err := ToRunwareImageGenerationRequest(&schemas.BifrostImageGenerationRequest{ + Model: "recraft:v4@vector", + Input: &schemas.ImageGenerationInput{Prompt: "a flat icon of a rocket"}, + Params: &schemas.ImageGenerationParameters{ + Type: new("vectorize"), + OutputFormat: new("svg"), + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if gen.TaskType != taskTypeVectorize { + t.Fatalf("generation taskType = %q, want %q", gen.TaskType, taskTypeVectorize) + } + if gen.OutputFormat == nil || *gen.OutputFormat != "SVG" { + t.Fatalf("outputFormat = %v, want SVG", gen.OutputFormat) + } + if gen.PositivePrompt == nil || *gen.PositivePrompt != "a flat icon of a rocket" { + t.Fatalf("prompt not carried: %v", gen.PositivePrompt) + } +} + +// Output asset UUIDs are surfaced per entry, resolved from whichever output family applies. +func TestToBifrostImageGenerationResponse_AssetIDs(t *testing.T) { + out, err := ToBifrostImageGenerationResponse(&RunwareResponse{Data: []RunwareResult{ + {ImageUUID: "img-1", ImageURL: "https://x/a.png"}, + {MaskImageUUID: "mask-1", MaskImageURL: "https://x/m.png"}, + {GuideImageUUID: "guide-1", GuideImageURL: "https://x/g.png"}, + }}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for i, want := range []string{"img-1", "mask-1", "guide-1"} { + if out.Data[i].ID != want { + t.Fatalf("data[%d].id = %q, want %q", i, out.Data[i].ID, want) + } + } +} + +// Runware resolves UUIDs and URLs itself, so a caller-supplied URL is forwarded untouched rather +// than being round-tripped through the gateway as base64. +func TestToRunwareImageEditRequest_ImageURL(t *testing.T) { + const src = "https://assets.runware.ai/covers/prunaai-p-image-upscale.jpg" + + upscale, err := ToRunwareImageEditRequest(&schemas.BifrostImageEditRequest{ + Model: "topazlabs:wonder@3.5", + Input: &schemas.ImageEditInput{Images: []schemas.ImageInput{{URL: src}}}, + Params: &schemas.ImageEditParameters{Type: new("upscale")}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if upscale.Inputs == nil || upscale.Inputs.Image == nil || *upscale.Inputs.Image != src { + t.Fatalf("inputs.image = %+v, want the URL verbatim", upscale.Inputs) + } + + // The imageInference path takes the same shortcut for its seed image. + edit, err := ToRunwareImageEditRequest(&schemas.BifrostImageEditRequest{ + Model: "runware:101@1", + Input: &schemas.ImageEditInput{ + Images: []schemas.ImageInput{{URL: src}}, + Prompt: "make it night", + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if edit.SeedImage == nil || *edit.SeedImage != src { + t.Fatalf("seedImage = %v, want the URL verbatim", edit.SeedImage) + } + + // Bytes still become a data URI, and an input with neither is rejected. + bytesReq, err := ToRunwareImageEditRequest(upscaleEditRequest(nil)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.HasPrefix(*bytesReq.Inputs.Image, "data:") { + t.Fatalf("expected a data URI for byte input, got %q", *bytesReq.Inputs.Image) + } + if _, err := ToRunwareImageEditRequest(&schemas.BifrostImageEditRequest{ + Model: "topazlabs:wonder@3.5", + Input: &schemas.ImageEditInput{Images: []schemas.ImageInput{{}}}, + }); err == nil { + t.Fatalf("expected an empty image input to be rejected") + } +} diff --git a/core/providers/runware/models.go b/core/providers/runware/models.go new file mode 100644 index 00000000000..c7d204fd3ad --- /dev/null +++ b/core/providers/runware/models.go @@ -0,0 +1,28 @@ +package runware + +// runware3DImageInputIsArray records, per 3D model, whether the input image goes into +// inputs.images[] (true) or inputs.image (false). The split is per-model and comes from Runware's +// published request schemas (https://schemas.runware.ai/resolve/); refresh this table when +// Runware adds a 3D model. Models absent from it fall back to the array form, which the majority +// of current models use. +// +// Runware silently drops inputs it does not recognise, so sending the wrong form yields a +// prompt-only result at full price instead of an error. Callers can override the whole object +// through the "inputs" extra param. +var runware3DImageInputIsArray = map[string]bool{ + "hyper3d:rodin@gen-2": true, + "meshy:meshy@6": true, + "meta:sam@3d": false, + "microsoft:trellis-2@4b": false, + "tencent:hunyuan-3d@3.1-pro": true, + "tencent:hunyuan-3d@3.1-rapid": false, + "tripo:v3.1@0": true, +} + +// uses3DImageArrayInput reports whether a 3D model takes its input image as inputs.images[]. +func uses3DImageArrayInput(model string) bool { + if isArray, ok := runware3DImageInputIsArray[model]; ok { + return isArray + } + return true +} diff --git a/core/providers/runware/runware.go b/core/providers/runware/runware.go index ea39e706f55..2efd986c966 100644 --- a/core/providers/runware/runware.go +++ b/core/providers/runware/runware.go @@ -323,7 +323,7 @@ func (provider *RunwareProvider) VideoGeneration(ctx *schemas.BifrostContext, ke return nil, providerUtils.EnrichError(ctx, bifrostErr, reqBody, respBody, sendBackRawRequest, sendBackRawResponse, latency) } - bifrostResp := ToBifrostVideoGenerationResponse(result) + bifrostResp := ToBifrostVideoGenerationResponse(result, videoResp.Errors) bifrostResp.ID = providerUtils.AddVideoIDProviderSuffix(result.TaskUUID, providerName) bifrostResp.Model = bifrostReq.Model bifrostResp.ExtraFields.Latency = latency.Milliseconds() @@ -365,7 +365,7 @@ func (provider *RunwareProvider) VideoRetrieve(ctx *schemas.BifrostContext, key return nil, providerUtils.EnrichError(ctx, bifrostErr, reqBody, respBody, sendBackRawRequest, sendBackRawResponse, latency) } - bifrostResp := ToBifrostVideoGenerationResponse(result) + bifrostResp := ToBifrostVideoGenerationResponse(result, videoResp.Errors) bifrostResp.ID = providerUtils.AddVideoIDProviderSuffix(taskID, providerName) bifrostResp.ExtraFields.Latency = latency.Milliseconds() if sendBackRawRequest { diff --git a/core/providers/runware/types.go b/core/providers/runware/types.go index 51fc3f687e2..d92204fe25f 100644 --- a/core/providers/runware/types.go +++ b/core/providers/runware/types.go @@ -12,6 +12,14 @@ const ( taskType3DInference = "3dInference" // taskTypeGetResponse polls an async task (e.g. video, 3D) by its taskUUID. taskTypeGetResponse = "getResponse" + // taskTypeUpscale enlarges an existing image; the model selects the upscaler. + taskTypeUpscale = "upscale" + // taskTypeRemoveBackground cuts the subject out onto a transparent background. + taskTypeRemoveBackground = "removeBackground" + // taskTypeImageMasking segments an image, returning a mask plus the regions it detected. + taskTypeImageMasking = "imageMasking" + // taskTypeVectorize converts a raster image, or a prompt, into an SVG. + taskTypeVectorize = "vectorize" ) // deliveryMethodAsync queues a task instead of holding the connection open; used for video. @@ -23,6 +31,14 @@ type RunwareFrameImage struct { Frame *string `json:"frame,omitempty"` // "first" | "last" } +// RunwareInputs holds a task's media inputs. Runware nests them under "inputs" while scalar +// parameters stay top-level. +type RunwareInputs struct { + Image *string `json:"image,omitempty"` // image UUID, URL, or base64/data-URI string + Images []string `json:"images,omitempty"` // array form, used by some 3D models + Video *string `json:"video,omitempty"` // video UUID or URL +} + // RunwareInferenceRequest is a single Runware task. taskType selects the operation; each // operation populates only the subset of fields it needs. Runware accepts an array of these // objects per request; the provider wraps a single task in an array before sending. @@ -51,6 +67,17 @@ type RunwareInferenceRequest struct { FrameImages []RunwareFrameImage `json:"frameImages,omitempty"` // image-to-video ReferenceImages []string `json:"referenceImages,omitempty"` + // Nested envelope, shared by the tool task types (upscale, removeBackground, 3D, ...). + Inputs *RunwareInputs `json:"inputs,omitempty"` + Settings map[string]interface{} `json:"settings,omitempty"` // model-specific tuning + ProviderSettings map[string]interface{} `json:"providerSettings,omitempty"` // keyed by vendor + OutputQuality *int `json:"outputQuality,omitempty"` + IncludeCost *bool `json:"includeCost,omitempty"` + + // Upscale-only. UpscaleFactor and TargetMegapixels are mutually exclusive. + UpscaleFactor *int `json:"upscaleFactor,omitempty"` + TargetMegapixels *int `json:"targetMegapixels,omitempty"` + // ExtraParams carries provider-native fields with no Bifrost equivalent // (CFGScale, scheduler, strength, maskMargin, outpaint, fps, lora, ...). Merged into // the request body by the transport layer when passthrough is enabled. @@ -95,11 +122,32 @@ type RunwareResult struct { VideoUUID string `json:"videoUUID,omitempty"` VideoURL string `json:"videoURL,omitempty"` + // Masking and ControlNet preprocessing name their artifact after what they produce rather + // than reusing the image* fields, and echo the input they were derived from. + MaskImageUUID string `json:"maskImageUUID,omitempty"` + MaskImageURL string `json:"maskImageURL,omitempty"` + MaskImageBase64Data string `json:"maskImageBase64Data,omitempty"` + MaskImageDataURI string `json:"maskImageDataURI,omitempty"` + GuideImageUUID string `json:"guideImageUUID,omitempty"` + GuideImageURL string `json:"guideImageURL,omitempty"` + GuideImageBase64Data string `json:"guideImageBase64Data,omitempty"` + GuideImageDataURI string `json:"guideImageDataURI,omitempty"` + InputImageUUID string `json:"inputImageUUID,omitempty"` + Detections []RunwareDetection `json:"detections,omitempty"` + // 3D / other modalities: newer Runware task types (e.g. 3dInference) return their // artifacts under a generic outputs.files[] array rather than modality-specific fields. Outputs *RunwareOutputs `json:"outputs,omitempty"` } +// RunwareDetection is a region located by an imageMasking model, in absolute input-image pixels. +type RunwareDetection struct { + XMin int `json:"x_min"` + YMin int `json:"y_min"` + XMax int `json:"x_max"` + YMax int `json:"y_max"` +} + // RunwareOutputs holds the generic artifact list returned by task types such as 3dInference. type RunwareOutputs struct { Files []RunwareOutputFile `json:"files,omitempty"` diff --git a/core/providers/runware/utils.go b/core/providers/runware/utils.go index 0c269e6d505..88405ac5ed6 100644 --- a/core/providers/runware/utils.go +++ b/core/providers/runware/utils.go @@ -3,6 +3,8 @@ package runware import ( "strconv" "strings" + + "github.com/bytedance/sonic" ) // Runware requires explicit pixel dimensions; default when the caller omits a size. @@ -52,26 +54,49 @@ func runwareOutputType(responseFormat *string) *string { return &out } -// runwareOutputFormat maps a Bifrost output_format to Runware's outputFormat enum. -// Returns nil to let Runware use its default. +// runwareOutputFormat maps a Bifrost output_format to Runware's outputFormat enum, which is +// uppercase and spans both image and video containers. Returns nil to let Runware use its default. func runwareOutputFormat(outputFormat *string) *string { if outputFormat == nil { return nil } var out string - switch strings.ToLower(*outputFormat) { + switch strings.ToLower(strings.TrimSpace(*outputFormat)) { case "png": out = "PNG" case "jpeg", "jpg": out = "JPG" case "webp": out = "WEBP" + case "svg": + out = "SVG" + case "mp4": + out = "MP4" + case "webm": + out = "WEBM" + case "mov": + out = "MOV" default: return nil } return &out } +// runwareSettings coerces a "settings" extra param into Runware's nested settings object. +// Multipart form values arrive as a JSON string; JSON callers send an object directly. +func runwareSettings(value interface{}) (map[string]interface{}, bool) { + switch v := value.(type) { + case map[string]interface{}: + return v, len(v) > 0 + case string: + var settings map[string]interface{} + if err := sonic.Unmarshal([]byte(v), &settings); err == nil && len(settings) > 0 { + return settings, true + } + } + return nil, false +} + // contentTypeForAssetURL infers a MIME type from an artifact URL's file extension. Runware's // output URLs (outputs.files[].url) carry the real extension (e.g. ".glb"), so the extension is // the most reliable signal — the request-side outputFormat parameter varies per model and is not @@ -103,6 +128,8 @@ func contentTypeForAssetURL(url string) string { return "video/mp4" case "webm": return "video/webm" + case "mov": + return "video/quicktime" default: return "application/octet-stream" } diff --git a/core/providers/runware/videos.go b/core/providers/runware/videos.go index 5deaa923993..026a5524b9d 100644 --- a/core/providers/runware/videos.go +++ b/core/providers/runware/videos.go @@ -19,25 +19,23 @@ import ( // the video task type; other task types (3D uses "resolution", etc.) supply their own dimensions // through extra_params. func ToRunwareVideoGenerationRequest(bifrostReq *schemas.BifrostVideoGenerationRequest) (*RunwareInferenceRequest, error) { + // Resolve the task type before building the request; it decides which modality-specific + // defaults apply below. + taskType := runwareVideoTaskType(bifrostReq.Params) + isVideo := taskType == taskTypeVideoInference + // Tool task types operate on an existing video rather than generating one. + isVideoTool := taskType == taskTypeUpscale || taskType == taskTypeRemoveBackground + if bifrostReq.Input == nil { return nil, fmt.Errorf("input is required") } - // Resolve the task type from extra_params before building the request; it decides which - // modality-specific defaults apply below. - taskType := taskTypeVideoInference - if bifrostReq.Params != nil { - if override, ok := schemas.SafeExtractString(bifrostReq.Params.ExtraParams["taskType"]); ok && override != "" { - taskType = override - } - } - isVideo := taskType == taskTypeVideoInference - request := &RunwareInferenceRequest{ TaskType: taskType, TaskUUID: uuid.New().String(), DeliveryMethod: new(deliveryMethodAsync), Model: bifrostReq.Model, + IncludeCost: new(true), } // Runware requires explicit width/height for video and rejects square sizes on some models; @@ -51,13 +49,27 @@ func ToRunwareVideoGenerationRequest(bifrostReq *schemas.BifrostVideoGenerationR request.PositivePrompt = &bifrostReq.Input.Prompt } - // Input reference image (image-to-video): anchored to the first frame. - if bifrostReq.Input.InputReference != nil && *bifrostReq.Input.InputReference != "" { + // Input asset. Tool tasks operate on the source video; 3D takes the reference image as a + // nested input in the singular or array form the model expects; video generation anchors it + // to the first frame. + switch { + case isVideoTool: + if bifrostReq.Input.VideoURI != nil && *bifrostReq.Input.VideoURI != "" { + request.Inputs = &RunwareInputs{Video: bifrostReq.Input.VideoURI} + } + case bifrostReq.Input.InputReference != nil && *bifrostReq.Input.InputReference != "": sanitizedURL, err := schemas.SanitizeImageURL(*bifrostReq.Input.InputReference) if err != nil { return nil, fmt.Errorf("invalid input reference: %w", err) } - request.FrameImages = []RunwareFrameImage{{InputImage: sanitizedURL, Frame: new("first")}} + switch { + case taskType != taskType3DInference: + request.FrameImages = []RunwareFrameImage{{InputImage: sanitizedURL, Frame: new("first")}} + case uses3DImageArrayInput(bifrostReq.Model): + request.Inputs = &RunwareInputs{Images: []string{sanitizedURL}} + default: + request.Inputs = &RunwareInputs{Image: &sanitizedURL} + } } if bifrostReq.Params != nil { @@ -80,16 +92,74 @@ func ToRunwareVideoGenerationRequest(bifrostReq *schemas.BifrostVideoGenerationR } request.ExtraParams = params.ExtraParams + + // Promote the Runware-native fields to typed properties so they reach the wire without + // extra-param passthrough, and drop them from ExtraParams so they are not sent twice. + if v, ok := runwareSettings(request.ExtraParams["settings"]); ok { + delete(request.ExtraParams, "settings") + request.Settings = v + } + if v, ok := runwareSettings(request.ExtraParams["providerSettings"]); ok { + delete(request.ExtraParams, "providerSettings") + request.ProviderSettings = v + } + request.UpscaleFactor = params.UpscaleFactor + request.TargetMegapixels = params.TargetMegapixels + // The /videos schema carries no output_format, so the container (MP4/WEBM/MOV) is only + // selectable through extra params. Video background removal needs an alpha-capable one. + if v, ok := schemas.SafeExtractStringPointer(request.ExtraParams["outputFormat"]); ok { + if format := runwareOutputFormat(v); format != nil { + delete(request.ExtraParams, "outputFormat") + request.OutputFormat = format + } + } } return request, nil } +// runwareVideoTaskType resolves the Runware task type for a /videos request. The neutral "type" +// parameter selects the operation, the taskType extra param stays as a raw escape hatch for task +// types Bifrost does not model, and video generation is the default. +func runwareVideoTaskType(params *schemas.VideoGenerationParameters) string { + if params == nil { + return taskTypeVideoInference + } + if params.Type != nil { + switch strings.ReplaceAll(strings.ToLower(strings.TrimSpace(*params.Type)), "-", "_") { + case "3d": + return taskType3DInference + case "upscale": + return taskTypeUpscale + case "background_removal", "remove_background", "remove_bg": + return taskTypeRemoveBackground + } + } + if override, ok := schemas.SafeExtractString(params.ExtraParams["taskType"]); ok && override != "" { + return override + } + return taskTypeVideoInference +} + +// findRunwareTaskError returns the envelope error belonging to a task, matching on taskUUID and +// falling back to the first error when the envelope does not attribute one. +func findRunwareTaskError(taskErrors []RunwareError, taskUUID string) *RunwareError { + for i := range taskErrors { + if taskErrors[i].TaskUUID == taskUUID { + return &taskErrors[i] + } + } + if len(taskErrors) > 0 { + return &taskErrors[0] + } + return nil +} + // ToBifrostVideoGenerationResponse converts a Runware task result to a Bifrost video response. // It handles both video tasks (videoURL) and other async task types that return artifacts under // outputs.files[] (e.g. 3dInference), surfacing every asset as a VideoOutput URL so callers can // consume them through the existing /videos response shape. -func ToBifrostVideoGenerationResponse(result *RunwareResult) *schemas.BifrostVideoGenerationResponse { +func ToBifrostVideoGenerationResponse(result *RunwareResult, taskErrors []RunwareError) *schemas.BifrostVideoGenerationResponse { response := &schemas.BifrostVideoGenerationResponse{ ID: result.TaskUUID, Object: "video", @@ -103,16 +173,33 @@ func ToBifrostVideoGenerationResponse(result *RunwareResult) *schemas.BifrostVid response.Status = schemas.VideoStatusInProgress case "error": response.Status = schemas.VideoStatusFailed - response.Error = &schemas.VideoCreateError{Code: result.Status, Message: "runware video task failed"} + // Runware reports the reason in the envelope's errors[], not on the task result, so a + // failed job would otherwise surface with no actionable detail. + response.Error = &schemas.VideoCreateError{Code: "error", Message: "runware task failed"} + if taskErr := findRunwareTaskError(taskErrors, result.TaskUUID); taskErr != nil { + if taskErr.Code != "" { + response.Error.Code = taskErr.Code + } + if taskErr.Message != "" { + response.Error.Message = taskErr.Message + } + } default: response.Status = schemas.VideoStatusQueued } if result.VideoURL != "" { + // outputFormat accepts MP4, WEBM and MOV, so derive the type from the URL rather than + // assuming MP4; fall back to MP4 for URLs that carry no usable extension. + contentType := contentTypeForAssetURL(result.VideoURL) + if contentType == "application/octet-stream" { + contentType = "video/mp4" + } response.Videos = append(response.Videos, schemas.VideoOutput{ + ID: result.VideoUUID, Type: schemas.VideoOutputTypeURL, URL: new(result.VideoURL), - ContentType: "video/mp4", + ContentType: contentType, }) } @@ -124,6 +211,7 @@ func ToBifrostVideoGenerationResponse(result *RunwareResult) *schemas.BifrostVid continue } response.Videos = append(response.Videos, schemas.VideoOutput{ + ID: file.UUID, Type: schemas.VideoOutputTypeURL, URL: new(file.URL), ContentType: contentTypeForAssetURL(file.URL), diff --git a/core/providers/runware/videos_test.go b/core/providers/runware/videos_test.go index 7b6882a3b3d..d413f9bbc6a 100644 --- a/core/providers/runware/videos_test.go +++ b/core/providers/runware/videos_test.go @@ -59,6 +59,203 @@ func TestToRunwareVideoGenerationRequest_3DOmitsWidthHeight(t *testing.T) { } } +// type="3d" selects the 3D task without the caller knowing Runware's task type names, and pulls +// settings out of extra params so model tuning reaches the wire as a nested object. +func TestToRunwareVideoGenerationRequest_3DTypeParam(t *testing.T) { + req := &schemas.BifrostVideoGenerationRequest{ + Model: "tencent:hunyuan-3d@3.1-rapid", + Input: &schemas.VideoGenerationInput{Prompt: "a ceramic teapot"}, + Params: &schemas.VideoGenerationParameters{ + Type: new("3d"), + ExtraParams: map[string]any{"settings": `{"pbr":true}`}, + }, + } + + out, err := ToRunwareVideoGenerationRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.TaskType != taskType3DInference { + t.Fatalf("taskType = %q, want %q", out.TaskType, taskType3DInference) + } + if out.Settings["pbr"] != true { + t.Fatalf("settings = %+v, want pbr=true", out.Settings) + } + if _, ok := out.ExtraParams["settings"]; ok { + t.Fatalf("settings should be consumed from ExtraParams, got %+v", out.ExtraParams) + } + if out.IncludeCost == nil || !*out.IncludeCost { + t.Fatalf("3D has no datasheet rate, so includeCost must be set") + } +} + +// Image-to-3D routes the reference image into the nested inputs object rather than frameImages, +// which the 3D task type does not accept. The singular/array form is per-model. +func TestToRunwareVideoGenerationRequest_3DInputArity(t *testing.T) { + build := func(model string) *RunwareInferenceRequest { + t.Helper() + out, err := ToRunwareVideoGenerationRequest(&schemas.BifrostVideoGenerationRequest{ + Model: model, + Input: &schemas.VideoGenerationInput{InputReference: new("https://assets.runware.ai/a.jpg")}, + Params: &schemas.VideoGenerationParameters{ + Type: new("3d"), + }, + }) + if err != nil { + t.Fatalf("unexpected error for %s: %v", model, err) + } + if len(out.FrameImages) != 0 { + t.Fatalf("%s: 3D must not use frameImages, got %+v", model, out.FrameImages) + } + return out + } + + singular := build("tencent:hunyuan-3d@3.1-rapid") + if singular.Inputs == nil || singular.Inputs.Image == nil || len(singular.Inputs.Images) != 0 { + t.Fatalf("rapid expects inputs.image, got %+v", singular.Inputs) + } + + array := build("tencent:hunyuan-3d@3.1-pro") + if array.Inputs == nil || len(array.Inputs.Images) != 1 || array.Inputs.Image != nil { + t.Fatalf("pro expects inputs.images[], got %+v", array.Inputs) + } +} + +// Video generation is unaffected: no type means videoInference, and the reference image still +// anchors to the first frame. +func TestToRunwareVideoGenerationRequest_VideoInputUnchanged(t *testing.T) { + out, err := ToRunwareVideoGenerationRequest(&schemas.BifrostVideoGenerationRequest{ + Model: "klingai:kling-video@3-pro", + Input: &schemas.VideoGenerationInput{ + Prompt: "a red bird flying", + InputReference: new("https://assets.runware.ai/a.jpg"), + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.TaskType != taskTypeVideoInference { + t.Fatalf("taskType = %q, want %q", out.TaskType, taskTypeVideoInference) + } + if len(out.FrameImages) != 1 || out.FrameImages[0].Frame == nil || *out.FrameImages[0].Frame != "first" { + t.Fatalf("video must keep frameImages anchoring, got %+v", out.FrameImages) + } + if out.Inputs != nil { + t.Fatalf("video must not use the nested inputs object, got %+v", out.Inputs) + } + if out.IncludeCost == nil || !*out.IncludeCost { + t.Fatalf("expected includeCost=true so the task cost is reported") + } +} + +// Video tool tasks (upscale, removeBackground) take their source from video_uri, nested under +// inputs.video, and never use frameImages or the video width/height defaults. +func TestToRunwareVideoGenerationRequest_VideoTools(t *testing.T) { + cases := map[string]string{ + "upscale": taskTypeUpscale, + "background_removal": taskTypeRemoveBackground, + "remove-bg": taskTypeRemoveBackground, + } + for editType, wantTask := range cases { + out, err := ToRunwareVideoGenerationRequest(&schemas.BifrostVideoGenerationRequest{ + Model: "bytedance:50@1", + Input: &schemas.VideoGenerationInput{VideoURI: new("https://assets.runware.ai/clip.mp4")}, + Params: &schemas.VideoGenerationParameters{ + Type: &editType, + UpscaleFactor: new(2), + ExtraParams: map[string]any{ + "providerSettings": `{"bria":{"preserveAlpha":true}}`, + }, + }, + }) + if err != nil { + t.Fatalf("%s: unexpected error: %v", editType, err) + } + if out.TaskType != wantTask { + t.Fatalf("%s: taskType = %q, want %q", editType, out.TaskType, wantTask) + } + if out.Inputs == nil || out.Inputs.Video == nil || *out.Inputs.Video != "https://assets.runware.ai/clip.mp4" { + t.Fatalf("%s: expected inputs.video, got %+v", editType, out.Inputs) + } + if len(out.FrameImages) != 0 || out.Width != nil || out.Height != nil { + t.Fatalf("%s: video-generation fields must stay unset, got %+v", editType, out) + } + if out.UpscaleFactor == nil || *out.UpscaleFactor != 2 { + t.Fatalf("%s: upscaleFactor = %v, want 2", editType, out.UpscaleFactor) + } + if out.ProviderSettings["bria"] == nil { + t.Fatalf("%s: providerSettings = %+v, want a bria entry", editType, out.ProviderSettings) + } + if out.DeliveryMethod == nil || *out.DeliveryMethod != deliveryMethodAsync { + t.Fatalf("%s: video tools are async-only, got %v", editType, out.DeliveryMethod) + } + } +} + +// video_uri alone is a complete request for a tool task: no prompt and no reference image. +func TestToRunwareVideoGenerationRequest_VideoURIOnly(t *testing.T) { + out, err := ToRunwareVideoGenerationRequest(&schemas.BifrostVideoGenerationRequest{ + Model: "bytedance:50@1", + Input: &schemas.VideoGenerationInput{VideoURI: new("https://assets.runware.ai/clip.mp4")}, + Params: &schemas.VideoGenerationParameters{ + Type: new("upscale"), + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Inputs == nil || out.Inputs.Video == nil { + t.Fatalf("expected inputs.video, got %+v", out.Inputs) + } + + // Generation still requires an input block. + if _, err := ToRunwareVideoGenerationRequest(&schemas.BifrostVideoGenerationRequest{ + Model: "klingai:kling-video@3-pro", + }); err == nil { + t.Fatalf("expected video generation without input to fail") + } +} + +// The /videos schema has no output_format, so the container is selected through extra params and +// normalised to Runware's uppercase enum. Video background removal is rejected unless it gets an +// alpha-capable container, so this is the only way to reach it. +func TestToRunwareVideoGenerationRequest_OutputFormat(t *testing.T) { + out, err := ToRunwareVideoGenerationRequest(&schemas.BifrostVideoGenerationRequest{ + Model: "bria:51@1", + Input: &schemas.VideoGenerationInput{VideoURI: new("https://assets.runware.ai/clip.mp4")}, + Params: &schemas.VideoGenerationParameters{ + Type: new("background_removal"), + ExtraParams: map[string]any{"outputFormat": "webm"}, + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.OutputFormat == nil || *out.OutputFormat != "WEBM" { + t.Fatalf("outputFormat = %v, want WEBM", out.OutputFormat) + } + if _, ok := out.ExtraParams["outputFormat"]; ok { + t.Fatalf("outputFormat should be consumed from ExtraParams, got %+v", out.ExtraParams) + } +} + +// outputFormat accepts MP4, WEBM and MOV, so the asset content type follows the URL rather than +// being assumed to be MP4. Extensionless URLs keep the MP4 fallback. +func TestToBifrostVideoGenerationResponse_ContentTypeFollowsFormat(t *testing.T) { + cases := map[string]string{ + "https://vm.runware.ai/v/clip.mp4": "video/mp4", + "https://vm.runware.ai/v/clip.webm": "video/webm", + "https://vm.runware.ai/v/clip.mov": "video/quicktime", + "https://vm.runware.ai/v/clip": "video/mp4", + } + for url, want := range cases { + resp := ToBifrostVideoGenerationResponse(&RunwareResult{Status: "success", VideoURL: url}, nil) + if len(resp.Videos) != 1 || resp.Videos[0].ContentType != want { + t.Errorf("%s: content type = %+v, want %q", url, resp.Videos, want) + } + } +} + // A 3D task result exposes its glb asset under outputs.files[]; the mapper surfaces it as a // VideoOutput URL with the model/gltf-binary content type and a completed status. func TestToBifrostVideoGenerationResponse_3DOutputs(t *testing.T) { @@ -72,7 +269,7 @@ func TestToBifrostVideoGenerationResponse_3DOutputs(t *testing.T) { }, } - resp := ToBifrostVideoGenerationResponse(result) + resp := ToBifrostVideoGenerationResponse(result, nil) if resp.Status != schemas.VideoStatusCompleted { t.Fatalf("status = %q, want %q", resp.Status, schemas.VideoStatusCompleted) } @@ -100,7 +297,7 @@ func TestToBifrostVideoGenerationResponse_Cost(t *testing.T) { }, } - resp := ToBifrostVideoGenerationResponse(result) + resp := ToBifrostVideoGenerationResponse(result, nil) if resp.Usage == nil || resp.Usage.Cost == nil { t.Fatalf("expected provider-reported cost to be surfaced, got Usage=%+v", resp.Usage) } @@ -112,7 +309,7 @@ func TestToBifrostVideoGenerationResponse_Cost(t *testing.T) { // When Runware does not report a cost (includeCost omitted), no Usage is attached so pricing can // fall through to the datasheet. func TestToBifrostVideoGenerationResponse_NoCost(t *testing.T) { - resp := ToBifrostVideoGenerationResponse(&RunwareResult{TaskUUID: "no-cost", Status: "success", VideoURL: "https://x/clip"}) + resp := ToBifrostVideoGenerationResponse(&RunwareResult{TaskUUID: "no-cost", Status: "success", VideoURL: "https://x/clip"}, nil) if resp.Usage != nil { t.Fatalf("expected no Usage when cost is absent, got %+v", resp.Usage) } @@ -127,7 +324,7 @@ func TestToBifrostVideoGenerationResponse_VideoUnchanged(t *testing.T) { VideoURL: "https://im.runware.ai/video/os/clip", } - resp := ToBifrostVideoGenerationResponse(result) + resp := ToBifrostVideoGenerationResponse(result, nil) if resp.Status != schemas.VideoStatusCompleted { t.Fatalf("status = %q, want completed", resp.Status) } @@ -138,7 +335,7 @@ func TestToBifrostVideoGenerationResponse_VideoUnchanged(t *testing.T) { // Processing tasks with no assets yet stay in progress / queued rather than being marked complete. func TestToBifrostVideoGenerationResponse_ProcessingNoAssets(t *testing.T) { - resp := ToBifrostVideoGenerationResponse(&RunwareResult{Status: "processing"}) + resp := ToBifrostVideoGenerationResponse(&RunwareResult{Status: "processing"}, nil) if resp.Status != schemas.VideoStatusInProgress { t.Fatalf("status = %q, want in_progress", resp.Status) } @@ -166,3 +363,46 @@ func TestContentTypeForAssetURL(t *testing.T) { } } } + +// A failed task carries its reason in the envelope's errors[], not on the result, so the mapper +// must be given the envelope or the caller is left with a generic failure. +func TestToBifrostVideoGenerationResponse_FailureReason(t *testing.T) { + result := &RunwareResult{TaskUUID: "task-1", Status: "error"} + + t.Run("uses the matching envelope error", func(t *testing.T) { + resp := ToBifrostVideoGenerationResponse(result, []RunwareError{ + {TaskUUID: "other", Code: "wrongOne", Message: "not this"}, + {TaskUUID: "task-1", Code: "inferenceError", Message: "inference error occurred"}, + }) + if resp.Status != schemas.VideoStatusFailed { + t.Fatalf("status = %q, want failed", resp.Status) + } + if resp.Error == nil || resp.Error.Code != "inferenceError" || resp.Error.Message != "inference error occurred" { + t.Fatalf("error not surfaced from envelope: %+v", resp.Error) + } + }) + + t.Run("falls back to a generic message with no envelope errors", func(t *testing.T) { + resp := ToBifrostVideoGenerationResponse(result, nil) + if resp.Error == nil || resp.Error.Message == "" { + t.Fatalf("expected a fallback message, got %+v", resp.Error) + } + }) +} + +// Runware accepts asset UUIDs as inputs, so output UUIDs are surfaced to allow chaining tasks. +func TestToBifrostVideoGenerationResponse_AssetIDs(t *testing.T) { + video := ToBifrostVideoGenerationResponse(&RunwareResult{ + Status: "success", VideoUUID: "vid-1", VideoURL: "https://x/clip.mp4", + }, nil) + if len(video.Videos) != 1 || video.Videos[0].ID != "vid-1" { + t.Fatalf("video asset id not surfaced: %+v", video.Videos) + } + + model := ToBifrostVideoGenerationResponse(&RunwareResult{ + Outputs: &RunwareOutputs{Files: []RunwareOutputFile{{UUID: "glb-1", URL: "https://x/m.glb"}}}, + }, nil) + if len(model.Videos) != 1 || model.Videos[0].ID != "glb-1" { + t.Fatalf("3D asset id not surfaced: %+v", model.Videos) + } +} diff --git a/core/providers/runway/utils.go b/core/providers/runway/utils.go index ec267c316c4..77f012cdc1c 100644 --- a/core/providers/runway/utils.go +++ b/core/providers/runway/utils.go @@ -12,7 +12,7 @@ import ( // - /v1/video_to_video: when video URI is provided // - /v1/image_to_video: when image input reference is provided func getRunwayEndpoint(req *schemas.BifrostVideoGenerationRequest) string { - if req.Params != nil && req.Params.VideoURI != nil && *req.Params.VideoURI != "" { + if req.Input != nil && req.Input.VideoURI != nil && *req.Input.VideoURI != "" { return "/v1/video_to_video" } if req.Input != nil && req.Input.InputReference != nil && *req.Input.InputReference != "" { diff --git a/core/providers/runway/videos.go b/core/providers/runway/videos.go index 809a8a1038b..e8a63e6f40c 100644 --- a/core/providers/runway/videos.go +++ b/core/providers/runway/videos.go @@ -43,6 +43,16 @@ func ToRunwayVideoGenerationRequest(bifrostReq *schemas.BifrostVideoGenerationRe } } + // Match getRunwayEndpoint, which only routes to video-to-video for a non-empty URI: treating an + // empty one as present would both reject supported models and forward an empty videoUri on the + // image-to-video endpoint. + if bifrostReq.Input != nil && bifrostReq.Input.VideoURI != nil && *bifrostReq.Input.VideoURI != "" { + if !supportsVideoToVideo(bifrostReq.Model) { + return nil, fmt.Errorf("video_uri is not supported for model %s", bifrostReq.Model) + } + request.VideoURI = bifrostReq.Input.VideoURI + } + if bifrostReq.Params != nil { if bifrostReq.Params.Seconds != nil { seconds, err := strconv.Atoi(*bifrostReq.Params.Seconds) @@ -69,13 +79,6 @@ func ToRunwayVideoGenerationRequest(bifrostReq *schemas.BifrostVideoGenerationRe } } - if bifrostReq.Params.VideoURI != nil { - if !supportsVideoToVideo(bifrostReq.Model) { - return nil, fmt.Errorf("video_uri is not supported for model %s", bifrostReq.Model) - } - request.VideoURI = bifrostReq.Params.VideoURI - } - if bifrostReq.Params.ExtraParams != nil { request.ExtraParams = bifrostReq.Params.ExtraParams // Handle references for video-to-video generation diff --git a/core/providers/runway/videos_test.go b/core/providers/runway/videos_test.go index a3c6051958b..1fed2568ab9 100644 --- a/core/providers/runway/videos_test.go +++ b/core/providers/runway/videos_test.go @@ -114,3 +114,40 @@ func TestToRunwayVideoGenerationRequest_ContentModeration(t *testing.T) { assert.NotContains(t, result.ExtraParams, "content_moderation") }) } + +// An empty video_uri is not a video-to-video request. getRunwayEndpoint only routes to +// /v1/video_to_video for a non-empty URI, so treating an empty one as present would reject +// supported models and forward an empty videoUri on the image-to-video endpoint. +func TestToRunwayVideoGenerationRequest_EmptyVideoURI(t *testing.T) { + empty := "" + + t.Run("does not reject a model without video-to-video support", func(t *testing.T) { + req := makeVideoReq("gen4_turbo", nil) + req.Input.VideoURI = &empty + req.Input.InputReference = schemas.Ptr("https://example.com/frame.png") + + out, err := ToRunwayVideoGenerationRequest(req) + require.NoError(t, err) + assert.Nil(t, out.VideoURI, "empty video_uri must not be forwarded") + assert.Equal(t, "/v1/image_to_video", getRunwayEndpoint(req), "routing must stay image-to-video") + }) + + t.Run("a non-empty uri on an unsupported model still errors", func(t *testing.T) { + req := makeVideoReq("gen4_turbo", nil) + req.Input.VideoURI = schemas.Ptr("https://example.com/clip.mp4") + + _, err := ToRunwayVideoGenerationRequest(req) + require.Error(t, err) + }) + + t.Run("a non-empty uri on a supported model is forwarded", func(t *testing.T) { + req := makeVideoReq("gen4_aleph", nil) + req.Input.VideoURI = schemas.Ptr("https://example.com/clip.mp4") + + out, err := ToRunwayVideoGenerationRequest(req) + require.NoError(t, err) + require.NotNil(t, out.VideoURI) + assert.Equal(t, "https://example.com/clip.mp4", *out.VideoURI) + assert.Equal(t, "/v1/video_to_video", getRunwayEndpoint(req)) + }) +} diff --git a/core/schemas/images.go b/core/schemas/images.go index 4cc13cb89bb..6bda6b43e87 100644 --- a/core/schemas/images.go +++ b/core/schemas/images.go @@ -40,6 +40,7 @@ type ImageGenerationParameters struct { OutputCompression *int `json:"output_compression,omitempty"` // compression level (0-100%) OutputFormat *string `json:"output_format,omitempty"` // "png", "webp", "jpeg" Style *string `json:"style,omitempty"` // "natural", "vivid" + Type *string `json:"type,omitempty"` // operation selector, e.g. "vectorize" ResponseFormat *string `json:"response_format,omitempty"` // "url", "b64_json" Seed *int `json:"seed,omitempty"` // seed for image generation NegativePrompt *string `json:"negative_prompt,omitempty"` // negative prompt for image generation @@ -190,10 +191,21 @@ type ImageGenerationResponseParameters struct { } type ImageData struct { - URL string `json:"url,omitempty"` - B64JSON string `json:"b64_json,omitempty"` - RevisedPrompt string `json:"revised_prompt,omitempty"` - Index int `json:"index"` + ID string `json:"id,omitempty"` // provider-side asset identifier, when one is assigned + URL string `json:"url,omitempty"` + B64JSON string `json:"b64_json,omitempty"` + RevisedPrompt string `json:"revised_prompt,omitempty"` + Index int `json:"index"` + Detections []ImageDetection `json:"detections,omitempty"` +} + +// ImageDetection is a region located by a masking or segmentation model. Coordinates are absolute +// pixels in the input image's own coordinate space, not normalized. +type ImageDetection struct { + XMin int `json:"x_min"` + YMin int `json:"y_min"` + XMax int `json:"x_max"` + YMax int `json:"y_max"` } type ImageUsage struct { @@ -326,10 +338,11 @@ type ImageEditInput struct { type ImageInput struct { Image []byte `json:"image"` + URL string `json:"url,omitempty"` // alternative to Image; providers that require bytes reject it } type ImageEditParameters struct { - Type *string `json:"type,omitempty"` // "inpainting", "outpainting", "background_removal", "remove_background", "erase_object", "recolor", "search_replace", "control_sketch", "control_structure", "style_guide", "style_transfer", "upscale_fast", "upscale_creative", "upscale_conservative" + Type *string `json:"type,omitempty"` // "inpainting", "outpainting", "background_removal", "remove_background", "erase_object", "recolor", "search_replace", "control_sketch", "control_structure", "style_guide", "style_transfer", "upscale", "upscale_fast", "upscale_creative", "upscale_conservative", "mask", "segmentation", "vectorize" Background *string `json:"background,omitempty"` // "transparent", "opaque", "auto" InputFidelity *string `json:"input_fidelity,omitempty"` // "low", "high" Mask []byte `json:"mask,omitempty"` @@ -344,6 +357,8 @@ type ImageEditParameters struct { NegativePrompt *string `json:"negative_prompt,omitempty"` // negative prompt for image editing Seed *int `json:"seed,omitempty"` // seed for image editing NumInferenceSteps *int `json:"num_inference_steps,omitempty"` // number of inference steps + UpscaleFactor *int `json:"upscale_factor,omitempty"` // type "upscale": multiply each dimension by N + TargetMegapixels *int `json:"target_megapixels,omitempty"` // type "upscale": target output size; mutually exclusive with upscale_factor ExtraParams map[string]interface{} `json:"-"` } diff --git a/core/schemas/videos.go b/core/schemas/videos.go index 55e8912604b..7791636f2a2 100644 --- a/core/schemas/videos.go +++ b/core/schemas/videos.go @@ -31,7 +31,8 @@ type ContentFilterInfo struct { } type VideoOutput struct { - Type VideoOutputType `json:"type"` // "url" | "base64" + ID string `json:"id,omitempty"` // provider-side asset identifier, when one is assigned + Type VideoOutputType `json:"type"` // "url" | "base64" URL *string `json:"url,omitempty"` Base64Data *string `json:"base64,omitempty"` ContentType string `json:"content_type"` @@ -84,6 +85,7 @@ func (b *BifrostVideoGenerationRequest) GetExtraParams() map[string]interface{} type VideoGenerationInput struct { Prompt string `json:"prompt"` InputReference *string `json:"input_reference,omitempty"` // Primary image for image-to-video (OpenAI-compatible) + VideoURI *string `json:"video_uri,omitempty"` // Source video for video-to-video and video tool tasks } type VideoGenerationParameters struct { @@ -92,9 +94,13 @@ type VideoGenerationParameters struct { NegativePrompt *string `json:"negative_prompt,omitempty"` Seed *int `json:"seed,omitempty"` - VideoURI *string `json:"video_uri,omitempty"` // for video to video generation + Type *string `json:"type,omitempty"` // operation selector, e.g. "3d", "upscale" Audio *bool `json:"audio,omitempty"` ExtraParams map[string]any `json:"-"` + + // Upscale operations (type "upscale"); mutually exclusive. + UpscaleFactor *int `json:"upscale_factor,omitempty"` + TargetMegapixels *int `json:"target_megapixels,omitempty"` } // DefaultVideoDuration is the default video duration in seconds for Gemini/Vertex when not specified. diff --git a/core/utils.go b/core/utils.go index 88432fc6acc..cf014590899 100644 --- a/core/utils.go +++ b/core/utils.go @@ -727,7 +727,7 @@ func isPromptOptionalImageEditType(t *string) bool { normalized := strings.ToLower(strings.TrimSpace(*t)) normalized = strings.ReplaceAll(normalized, "-", "_") return slices.Contains( - []string{"background_removal", "remove_background", "remove_bg", "erase_object", "upscale_fast"}, + []string{"background_removal", "remove_background", "remove_bg", "erase_object", "upscale", "upscale_fast", "mask", "segmentation", "vectorize"}, normalized, ) } diff --git a/framework/modelcatalog/datasheet/cost.go b/framework/modelcatalog/datasheet/cost.go index eccff2ac8e8..6205e9e3f44 100644 --- a/framework/modelcatalog/datasheet/cost.go +++ b/framework/modelcatalog/datasheet/cost.go @@ -215,6 +215,14 @@ func (s *Store) calculateBaseCost(result *schemas.BifrostResponse, scopes Lookup } requestType := extraFields.RequestType + // A retrieve is a status read, not a generation. Providers that report the job's cost on the + // polled response (e.g. Runware, which echoes it on every getResponse once the task has + // succeeded) would otherwise be billed again on every poll, inflating logs, traces and + // governance budgets. + if requestType == schemas.VideoRetrieveRequest { + return 0 + } + // Extract usage data from the response (passthrough and native paths unified) input := extractCostInput(result) diff --git a/framework/modelcatalog/datasheet/cost_test.go b/framework/modelcatalog/datasheet/cost_test.go index 0291ded28db..7eafd737589 100644 --- a/framework/modelcatalog/datasheet/cost_test.go +++ b/framework/modelcatalog/datasheet/cost_test.go @@ -1919,6 +1919,25 @@ func TestCalculateCost_VideoProviderComputedCostPassthrough(t *testing.T) { assert.Equal(t, 0.5, s.CalculateCost(resp, nil)) } +// The same payload polled back through a retrieve must not be billed: Runware echoes the task cost +// on every getResponse once it has succeeded, so billing the retrieve would charge the job again +// on each poll. +func TestCalculateCost_VideoRetrieveIsNotBilled(t *testing.T) { + s := testStoreWithPricing(nil) + + resp := &schemas.BifrostResponse{ + VideoGenerationResponse: &schemas.BifrostVideoGenerationResponse{ + Usage: &schemas.VideoUsage{Cost: &schemas.BifrostCost{TotalCost: 0.5}}, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.VideoRetrieveRequest, + RoutingInfo: routingInfoFor(schemas.Runware, "tripo:v3.1@0"), + }, + }, + } + + assert.Zero(t, s.CalculateCost(resp, nil)) +} + // Passthrough responses can carry a provider-reported cost (e.g. Runware's per-task cost read from // the raw body). It must win verbatim, with no datasheet entry required. func TestCalculateCost_PassthroughProviderComputedCost(t *testing.T) { diff --git a/transports/bifrost-http/handlers/inference.go b/transports/bifrost-http/handlers/inference.go index ab0e98d71ba..35ceccf179e 100644 --- a/transports/bifrost-http/handlers/inference.go +++ b/transports/bifrost-http/handlers/inference.go @@ -269,6 +269,7 @@ var imageGenerationParamsKnownFields = map[string]bool{ "user": true, "aspect_ratio": true, "input_images": true, + "type": true, } // imageEditParamsKnownFields contains known fields for image edit requests @@ -279,6 +280,8 @@ var imageEditParamsKnownFields = map[string]bool{ "fallbacks": true, "image": true, "image[]": true, + "image_url": true, + "image_url[]": true, "mask": true, "type": true, "background": true, @@ -294,6 +297,8 @@ var imageEditParamsKnownFields = map[string]bool{ "negative_prompt": true, "seed": true, "num_inference_steps": true, + "upscale_factor": true, + "target_megapixels": true, "stream": true, } @@ -313,16 +318,19 @@ var imageVariationParamsKnownFields = map[string]bool{ // videoGenerationParamsKnownFields contains known fields for video generation requests // Based on VideoGenerationInput and VideoGenerationParameters structs var videoGenerationParamsKnownFields = map[string]bool{ - "model": true, - "prompt": true, - "input_reference": true, - "seconds": true, - "size": true, - "negative_prompt": true, - "seed": true, - "video_uri": true, - "audio": true, - "fallbacks": true, + "model": true, + "prompt": true, + "input_reference": true, + "seconds": true, + "size": true, + "negative_prompt": true, + "seed": true, + "type": true, + "upscale_factor": true, + "target_megapixels": true, + "video_uri": true, + "audio": true, + "fallbacks": true, } var videoRemixParamsKnownFields = map[string]bool{ @@ -2339,10 +2347,19 @@ func prepareImageEditRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (*Ima } else if imageFilesSingle := form.File["image"]; len(imageFilesSingle) > 0 { imageFiles = imageFilesSingle } - if len(imageFiles) == 0 { - return nil, nil, fmt.Errorf("at least one image is required") + // Providers whose upstream fetches the asset itself (e.g. Runware) accept a URL in place of an + // upload, which avoids round-tripping large images through the gateway as base64. + imageURLs := form.Value["image_url[]"] + if len(imageURLs) == 0 { + imageURLs = form.Value["image_url"] } - images := make([]schemas.ImageInput, 0, len(imageFiles)) + if len(imageFiles) == 0 && len(imageURLs) == 0 { + return nil, nil, fmt.Errorf("at least one image or image_url is required") + } + // Uploads keep their leading position so a request that sends no URL is ordered exactly as + // before: providers treat the first image as the primary one (Runware's seed image, Bedrock's + // style-transfer base), so the order is part of the contract. + images := make([]schemas.ImageInput, 0, len(imageFiles)+len(imageURLs)) for _, fh := range imageFiles { f, err := fh.Open() if err != nil { @@ -2355,6 +2372,11 @@ func prepareImageEditRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (*Ima } images = append(images, schemas.ImageInput{Image: fileData}) } + for _, imageURL := range imageURLs { + if imageURL != "" { + images = append(images, schemas.ImageInput{URL: imageURL}) + } + } prompt := "" if len(promptValues) > 0 && promptValues[0] != "" { prompt = promptValues[0] @@ -2400,6 +2422,20 @@ func prepareImageEditRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (*Ima } req.ImageEditParameters.NumInferenceSteps = &numInferenceSteps } + if upscaleFactorValues := form.Value["upscale_factor"]; len(upscaleFactorValues) > 0 && upscaleFactorValues[0] != "" { + upscaleFactor, err := strconv.Atoi(upscaleFactorValues[0]) + if err != nil { + return nil, nil, fmt.Errorf("invalid upscale_factor value: %v", err) + } + req.ImageEditParameters.UpscaleFactor = &upscaleFactor + } + if targetMegapixelsValues := form.Value["target_megapixels"]; len(targetMegapixelsValues) > 0 && targetMegapixelsValues[0] != "" { + targetMegapixels, err := strconv.Atoi(targetMegapixelsValues[0]) + if err != nil { + return nil, nil, fmt.Errorf("invalid target_megapixels value: %v", err) + } + req.ImageEditParameters.TargetMegapixels = &targetMegapixels + } if seedValues := form.Value["seed"]; len(seedValues) > 0 && seedValues[0] != "" { seed, err := strconv.Atoi(seedValues[0]) if err != nil { @@ -2663,15 +2699,17 @@ func (h *CompletionHandler) videoGeneration(ctx *fasthttp.RequestCtx) { return } - if req.VideoGenerationInput == nil || req.Prompt == "" { - SendError(ctx, fasthttp.StatusBadRequest, "prompt cannot be empty") - return - } - if req.VideoGenerationParameters == nil { req.VideoGenerationParameters = &schemas.VideoGenerationParameters{} } + // Operations driven by an input asset (video upscale, image-to-3D) carry no prompt. + if req.VideoGenerationInput == nil || + (req.Prompt == "" && req.InputReference == nil && req.VideoURI == nil) { + SendError(ctx, fasthttp.StatusBadRequest, "prompt, input_reference or video_uri is required") + return + } + extraParams, err := extractExtraParams(ctx.PostBody(), videoGenerationParamsKnownFields) if err != nil { logger.Warn("Failed to extract extra params: %v", err)