fix: runware upscale compatibility

core/bifrost.go

@@ -1905,11 +1905,13 @@ func (bifrost *Bifrost) VideoGenerationRequest(ctx *schemas.BifrostContext,
 			},
 		}
 	}
-	if req.Input == nil || req.Input.Prompt == "" {
+	// Operations driven by an input asset (video upscale, image-to-3D) carry no prompt; the
+	// provider rejects a combination its model cannot serve.
+	if req.Input == nil || (req.Input.Prompt == "" && req.Input.InputReference == nil && req.Input.VideoURI == nil) {
 		return nil, &schemas.BifrostError{
 			IsBifrostError: false,
 			Error: &schemas.ErrorField{
-				Message: "prompt not provided for video generation request",
+				Message: "prompt, input_reference or video_uri not provided for video generation request",
 			},
 			ExtraFields: schemas.BifrostErrorExtraFields{
 				RequestType:            schemas.VideoGenerationRequest,

core/providers/gemini/images.go

@@ -846,6 +846,9 @@ func ToGeminiImageEditRequest(bifrostReq *schemas.BifrostImageEditRequest) *Gemi
 	}
 
 	for _, image := range bifrostReq.Input.Images {
+		if len(image.Image) == 0 {
+			continue
+		}
 		// Detect MIME type from image bytes
 		mimeType := http.DetectContentType(image.Image)
 		// Fallback to PNG if detection fails

core/providers/gemini/videos.go

@@ -277,9 +277,9 @@ func ToGeminiVideoGenerationRequest(bifrostReq *schemas.BifrostVideoGenerationRe
 		instance.Image = image
 	}
 
-	if bifrostReq.Params != nil && bifrostReq.Params.VideoURI != nil {
+	if bifrostReq.Input != nil && bifrostReq.Input.VideoURI != nil {
 		instance.Video = &VideoGenerationVideoInput{
-			URI: bifrostReq.Params.VideoURI,
+			URI: bifrostReq.Input.VideoURI,
 		}
 	}
 
@@ -586,8 +586,7 @@ func (request *GeminiVideoGenerationRequest) ToBifrostVideoGenerationRequest(ctx
 
 	// Handle video URI
 	if instance.Video != nil && instance.Video.URI != nil {
-		ensureParams()
-		bifrostReq.Params.VideoURI = instance.Video.URI
+		bifrostReq.Input.VideoURI = instance.Video.URI
 	}
 
 	// Handle last frame

core/providers/openai/types.go

@@ -1134,8 +1134,9 @@ var ValidOpenAIVideoSizes = map[string]bool{
 
 // OpenAIVideoGenerationRequest is the request body for OpenAI video generation.
 type OpenAIVideoGenerationRequest struct {
-	Prompt         string `json:"prompt"`                    // Text prompt that describes the video to generate (max 32000, min 1)
-	InputReference []byte `json:"input_reference,omitempty"` // Optional image reference file that guides generation
+	Prompt         string  `json:"prompt"`                    // Text prompt that describes the video to generate (max 32000, min 1)
+	InputReference []byte  `json:"input_reference,omitempty"` // Optional image reference file that guides generation
+	VideoURI       *string `json:"video_uri,omitempty"`       // Optional source video for video-to-video
 
 	Model string `json:"model"` // Video generation model (defaults to sora-2)
 

core/providers/openai/videos.go

@@ -103,7 +103,8 @@ func (req *OpenAIVideoGenerationRequest) ToBifrostVideoGenerationRequest(ctx *sc
 	provider, model := schemas.ParseModelString(req.Model, "")
 
 	input := &schemas.VideoGenerationInput{
-		Prompt: req.Prompt,
+		Prompt:   req.Prompt,
+		VideoURI: req.VideoURI,
 	}
 	if req.InputReference != nil {
 		input.InputReference = schemas.Ptr(providerUtils.FileBytesToBase64DataURL(req.InputReference))

core/providers/runware/images.go

@@ -2,6 +2,7 @@ package runware
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/google/uuid"
 	providerUtils "github.com/maximhq/bifrost/core/providers/utils"
@@ -16,14 +17,22 @@ func ToRunwareImageGenerationRequest(bifrostReq *schemas.BifrostImageGenerationR
 		return nil, fmt.Errorf("input is required")
 	}
 
+	// Text-to-SVG runs as its own task type on this endpoint; everything else is imageInference.
+	taskType := taskTypeImageInference
+	if bifrostReq.Params != nil && bifrostReq.Params.Type != nil &&
+		strings.EqualFold(strings.TrimSpace(*bifrostReq.Params.Type), "vectorize") {
+		taskType = taskTypeVectorize
+	}
+
 	width, height := defaultRunwareWidth, defaultRunwareHeight
 	request := &RunwareInferenceRequest{
-		TaskType:       taskTypeImageInference,
+		TaskType:       taskType,
 		TaskUUID:       uuid.New().String(),
 		Model:          bifrostReq.Model,
 		PositivePrompt: &bifrostReq.Input.Prompt,
 		Width:          &width,
 		Height:         &height,
+		IncludeCost:    new(true),
 	}
 
 	if bifrostReq.Params != nil {
@@ -59,10 +68,14 @@ func ToRunwareImageEditRequest(bifrostReq *schemas.BifrostImageEditRequest) (*Ru
 	if bifrostReq.Input == nil {
 		return nil, fmt.Errorf("input is required")
 	}
-	if len(bifrostReq.Input.Images) == 0 || len(bifrostReq.Input.Images[0].Image) == 0 {
+	if len(bifrostReq.Input.Images) == 0 || runwareImageInput(bifrostReq.Input.Images[0]) == "" {
 		return nil, fmt.Errorf("at least one input image is required")
 	}
 
+	if taskType := runwareImageEditTaskType(bifrostReq.Params); taskType != "" {
+		return toRunwareImageToolRequest(taskType, bifrostReq)
+	}
+
 	width, height := defaultRunwareWidth, defaultRunwareHeight
 	request := &RunwareInferenceRequest{
 		TaskType:       taskTypeImageInference,
@@ -71,10 +84,11 @@ func ToRunwareImageEditRequest(bifrostReq *schemas.BifrostImageEditRequest) (*Ru
 		PositivePrompt: &bifrostReq.Input.Prompt,
 		Width:          &width,
 		Height:         &height,
+		IncludeCost:    new(true),
 	}
 
 	// Seed image: the base image being edited (raw bytes -> base64 data URI).
-	seedImage := providerUtils.FileBytesToBase64DataURL(bifrostReq.Input.Images[0].Image)
+	seedImage := runwareImageInput(bifrostReq.Input.Images[0])
 	request.SeedImage = &seedImage
 
 	if bifrostReq.Params != nil {
@@ -102,6 +116,95 @@ func ToRunwareImageEditRequest(bifrostReq *schemas.BifrostImageEditRequest) (*Ru
 	return request, nil
 }
 
+// runwareImageInput resolves an input image to the reference Runware expects. A caller-supplied
+// URL passes through untouched — Runware accepts UUIDs and URLs natively, so forwarding it avoids
+// round-tripping the asset through the gateway as base64 — while raw bytes become a data URI.
+func runwareImageInput(img schemas.ImageInput) string {
+	if img.URL != "" {
+		return img.URL
+	}
+	if len(img.Image) == 0 {
+		return ""
+	}
+	return providerUtils.FileBytesToBase64DataURL(img.Image)
+}
+
+// runwareImageEditTaskType maps the neutral edit type onto a Runware tool task type. An empty
+// result means the edit runs as a regular imageInference task (image-to-image, inpainting,
+// outpainting).
+func runwareImageEditTaskType(params *schemas.ImageEditParameters) string {
+	if params == nil || params.Type == nil {
+		return ""
+	}
+	switch strings.ReplaceAll(strings.ToLower(strings.TrimSpace(*params.Type)), "-", "_") {
+	case "upscale":
+		return taskTypeUpscale
+	case "background_removal", "remove_background", "remove_bg":
+		return taskTypeRemoveBackground
+	case "mask", "segmentation":
+		return taskTypeImageMasking
+	case "vectorize":
+		return taskTypeVectorize
+	}
+	return ""
+}
+
+// runwareResultAssets resolves a task result's output family. Runware names an artifact after what
+// the task produces, so masking returns maskImage* and ControlNet preprocessing returns
+// guideImage* rather than reusing image*; reading only image* would silently drop the output.
+func runwareResultAssets(result *RunwareResult) (id string, url string, base64Data string, dataURI string) {
+	switch {
+	case result.ImageURL != "", result.ImageBase64Data != "", result.ImageDataURI != "":
+		return result.ImageUUID, result.ImageURL, result.ImageBase64Data, result.ImageDataURI
+	case result.MaskImageURL != "", result.MaskImageBase64Data != "", result.MaskImageDataURI != "":
+		return result.MaskImageUUID, result.MaskImageURL, result.MaskImageBase64Data, result.MaskImageDataURI
+	case result.GuideImageURL != "", result.GuideImageBase64Data != "", result.GuideImageDataURI != "":
+		return result.GuideImageUUID, result.GuideImageURL, result.GuideImageBase64Data, result.GuideImageDataURI
+	}
+	return result.ImageUUID, "", "", ""
+}
+
+// toRunwareImageToolRequest builds a Runware single-image tool task (upscale, removeBackground).
+// These share one envelope: the image is nested under "inputs", model tuning goes in "settings"
+// and "providerSettings", and none of the imageInference fields (prompt, dimensions, steps) apply,
+// so they are left unset. Runware-native fields are read from extra params under their own names.
+func toRunwareImageToolRequest(taskType string, bifrostReq *schemas.BifrostImageEditRequest) (*RunwareInferenceRequest, error) {
+	image := runwareImageInput(bifrostReq.Input.Images[0])
+	request := &RunwareInferenceRequest{
+		TaskType:    taskType,
+		TaskUUID:    uuid.New().String(),
+		Model:       bifrostReq.Model,
+		Inputs:      &RunwareInputs{Image: &image},
+		IncludeCost: new(true),
+	}
+
+	if bifrostReq.Params == nil {
+		return request, nil
+	}
+	params := bifrostReq.Params
+
+	request.OutputType = runwareOutputType(params.ResponseFormat)
+	request.OutputFormat = runwareOutputFormat(params.OutputFormat)
+	request.OutputQuality = params.OutputCompression
+	request.ExtraParams = params.ExtraParams
+
+	// Consume the fields promoted to typed properties so they are not also re-sent verbatim
+	// when extra-param passthrough is enabled.
+	if v, ok := runwareSettings(request.ExtraParams["settings"]); ok {
+		delete(request.ExtraParams, "settings")
+		request.Settings = v
+	}
+	if v, ok := runwareSettings(request.ExtraParams["providerSettings"]); ok {
+		delete(request.ExtraParams, "providerSettings")
+		request.ProviderSettings = v
+	}
+
+	request.UpscaleFactor = params.UpscaleFactor
+	request.TargetMegapixels = params.TargetMegapixels
+
+	return request, nil
+}
+
 // ToBifrostImageGenerationResponse converts a Runware response envelope to a Bifrost image response.
 func ToBifrostImageGenerationResponse(resp *RunwareResponse) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) {
 	if resp == nil {
@@ -125,13 +228,23 @@ func ToBifrostImageGenerationResponse(resp *RunwareResponse) (*schemas.BifrostIm
 	var totalCost float64
 	for i, img := range resp.Data {
 		data := schemas.ImageData{Index: i}
+		id, url, base64Data, dataURI := runwareResultAssets(&img)
+		// Runware accepts these UUIDs as inputs, so surfacing them lets callers chain tasks
+		// (mask then inpaint, upscale then remove background) without re-uploading the asset.
+		data.ID = id
 		switch {
-		case img.ImageURL != "":
-			data.URL = img.ImageURL
-		case img.ImageBase64Data != "":
-			data.B64JSON = img.ImageBase64Data
-		case img.ImageDataURI != "":
-			data.URL = img.ImageDataURI
+		case url != "":
+			data.URL = url
+		case base64Data != "":
+			data.B64JSON = base64Data
+		case dataURI != "":
+			data.URL = dataURI
+		}
+		// Masking models report the regions they located alongside the mask itself.
+		for _, d := range img.Detections {
+			data.Detections = append(data.Detections, schemas.ImageDetection{
+				XMin: d.XMin, YMin: d.YMin, XMax: d.XMax, YMax: d.YMax,
+			})
 		}
 		bifrostResp.Data = append(bifrostResp.Data, data)
 		if img.Seed != nil {