diff --git a/common/endpoint_defaults.go b/common/endpoint_defaults.go index 11ec79217530..5c85fbb220f4 100644 --- a/common/endpoint_defaults.go +++ b/common/endpoint_defaults.go @@ -24,7 +24,9 @@ var defaultEndpointInfoMap = map[constant.EndpointType]EndpointInfo{ constant.EndpointTypeGemini: {Path: "/v1beta/models/{model}:generateContent", Method: "POST"}, constant.EndpointTypeJinaRerank: {Path: "/v1/rerank", Method: "POST"}, constant.EndpointTypeImageGeneration: {Path: "/v1/images/generations", Method: "POST"}, + constant.EndpointTypeImageEdit: {Path: "/v1/images/edits", Method: "POST"}, constant.EndpointTypeEmbeddings: {Path: "/v1/embeddings", Method: "POST"}, + constant.EndpointTypeOpenAIVideo: {Path: "/v1/video/generations", Method: "POST"}, } // GetDefaultEndpointInfo 返回指定端点类型的默认信息以及是否存在 diff --git a/common/endpoint_type.go b/common/endpoint_type.go index a5e2ff8412e8..51c564d1960f 100644 --- a/common/endpoint_type.go +++ b/common/endpoint_type.go @@ -37,7 +37,9 @@ func GetEndpointTypesByChannelType(channelType int, modelName string) []constant endpointTypes = []constant.EndpointType{constant.EndpointTypeOpenAI} } } - if IsImageGenerationModel(modelName) { + if IsImageEditModel(modelName) { + endpointTypes = append([]constant.EndpointType{constant.EndpointTypeImageEdit}, endpointTypes...) + } else if IsImageGenerationModel(modelName) { // add to first endpointTypes = append([]constant.EndpointType{constant.EndpointTypeImageGeneration}, endpointTypes...) } diff --git a/common/endpoint_type_test.go b/common/endpoint_type_test.go new file mode 100644 index 000000000000..0c44158a61f8 --- /dev/null +++ b/common/endpoint_type_test.go @@ -0,0 +1,53 @@ +package common + +import ( + "testing" + + "github.com/QuantumNous/new-api/constant" +) + +func TestGetEndpointTypesByChannelTypeRecognizesGrokImagineImageModels(t *testing.T) { + tests := []struct { + name string + channel int + model string + expected constant.EndpointType + }{ + { + name: "grok imagine 1.0 generation", + channel: constant.ChannelTypeXai, + model: "grok-imagine-1.0", + expected: constant.EndpointTypeImageGeneration, + }, + { + name: "grok imagine 1.0 fast generation", + channel: constant.ChannelTypeXai, + model: "grok-imagine-1.0-fast", + expected: constant.EndpointTypeImageGeneration, + }, + { + name: "grok imagine 1.0 edit", + channel: constant.ChannelTypeXai, + model: "grok-imagine-1.0-edit", + expected: constant.EndpointTypeImageEdit, + }, + { + name: "grok imagine video stays video", + channel: constant.ChannelTypeSora, + model: "grok-imagine-1.0-video", + expected: constant.EndpointTypeOpenAIVideo, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetEndpointTypesByChannelType(tt.channel, tt.model) + if len(got) == 0 { + t.Fatalf("expected endpoint types for %s", tt.model) + } + if got[0] != tt.expected { + t.Fatalf("expected first endpoint %s, got %s", tt.expected, got[0]) + } + }) + } +} diff --git a/common/model.go b/common/model.go index 4ebc7b532d74..035d421fe818 100644 --- a/common/model.go +++ b/common/model.go @@ -13,10 +13,15 @@ var ( "dall-e-3", "dall-e-2", "gpt-image-1", + "exact:grok-imagine-1.0", + "exact:grok-imagine-1.0-fast", "prefix:imagen-", "flux-", "flux.1-", } + ImageEditModels = []string{ + "exact:grok-imagine-1.0-edit", + } OpenAITextModels = []string{ "gpt-", "o1", @@ -38,6 +43,25 @@ func IsOpenAIResponseOnlyModel(modelName string) bool { func IsImageGenerationModel(modelName string) bool { modelName = strings.ToLower(modelName) for _, m := range ImageGenerationModels { + if strings.HasPrefix(m, "exact:") && modelName == strings.TrimPrefix(m, "exact:") { + return true + } + if strings.Contains(modelName, m) { + return true + } + if strings.HasPrefix(m, "prefix:") && strings.HasPrefix(modelName, strings.TrimPrefix(m, "prefix:")) { + return true + } + } + return false +} + +func IsImageEditModel(modelName string) bool { + modelName = strings.ToLower(modelName) + for _, m := range ImageEditModels { + if strings.HasPrefix(m, "exact:") && modelName == strings.TrimPrefix(m, "exact:") { + return true + } if strings.Contains(modelName, m) { return true } diff --git a/constant/endpoint_type.go b/constant/endpoint_type.go index 8681bf06e319..c5d1e3be3a97 100644 --- a/constant/endpoint_type.go +++ b/constant/endpoint_type.go @@ -10,6 +10,7 @@ const ( EndpointTypeGemini EndpointType = "gemini" EndpointTypeJinaRerank EndpointType = "jina-rerank" EndpointTypeImageGeneration EndpointType = "image-generation" + EndpointTypeImageEdit EndpointType = "image-edit" EndpointTypeEmbeddings EndpointType = "embeddings" EndpointTypeOpenAIVideo EndpointType = "openai-video" //EndpointTypeMidjourney EndpointType = "midjourney-proxy" diff --git a/controller/playground.go b/controller/playground.go index 9e015b26d5d9..9fa64f09e492 100644 --- a/controller/playground.go +++ b/controller/playground.go @@ -57,6 +57,36 @@ func PlaygroundVideoSubmit(c *gin.Context) { RelayTask(c) } +func PlaygroundImageGenerations(c *gin.Context) { + var newAPIError *types.NewAPIError + defer func() { + if newAPIError != nil { + c.JSON(newAPIError.StatusCode, gin.H{ + "error": newAPIError.ToOpenAIError(), + }) + } + }() + if newAPIError = setupPlaygroundTokenContext(c, "playground-image", c.GetString("group")); newAPIError != nil { + return + } + Relay(c, types.RelayFormatOpenAIImage) +} + +func PlaygroundImageEdits(c *gin.Context) { + var newAPIError *types.NewAPIError + defer func() { + if newAPIError != nil { + c.JSON(newAPIError.StatusCode, gin.H{ + "error": newAPIError.ToOpenAIError(), + }) + } + }() + if newAPIError = setupPlaygroundTokenContext(c, "playground-image-edit", c.GetString("group")); newAPIError != nil { + return + } + Relay(c, types.RelayFormatOpenAIImage) +} + func PlaygroundVideoFetch(c *gin.Context) { var newAPIError *types.NewAPIError defer func() { diff --git a/controller/relay.go b/controller/relay.go index 10dfd502fbd0..030cecf00c7d 100644 --- a/controller/relay.go +++ b/controller/relay.go @@ -16,6 +16,7 @@ import ( "github.com/QuantumNous/new-api/middleware" "github.com/QuantumNous/new-api/model" "github.com/QuantumNous/new-api/relay" + taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon" relaycommon "github.com/QuantumNous/new-api/relay/common" relayconstant "github.com/QuantumNous/new-api/relay/constant" "github.com/QuantumNous/new-api/relay/helper" @@ -586,6 +587,42 @@ func RelayTask(c *gin.Context) { task.Quota = result.Quota task.Data = result.TaskData task.Action = relayInfo.Action + if adaptor := relay.GetTaskAdaptor(result.Platform); adaptor != nil && len(result.TaskData) > 0 { + if taskInfo, err := adaptor.ParseTaskResult(result.TaskData); err == nil && taskInfo != nil && taskInfo.Status != "" { + now := time.Now().Unix() + task.Status = model.TaskStatus(taskInfo.Status) + switch task.Status { + case model.TaskStatusSubmitted: + task.Progress = taskcommon.ProgressSubmitted + case model.TaskStatusQueued: + task.Progress = taskcommon.ProgressQueued + case model.TaskStatusInProgress: + task.Progress = taskcommon.ProgressInProgress + if task.StartTime == 0 { + task.StartTime = now + } + case model.TaskStatusSuccess: + task.Progress = taskcommon.ProgressComplete + if task.StartTime == 0 { + task.StartTime = now + } + if task.FinishTime == 0 { + task.FinishTime = now + } + task.PrivateData.ResultURL = taskInfo.Url + case model.TaskStatusFailure: + task.Progress = taskcommon.ProgressComplete + if task.FinishTime == 0 { + task.FinishTime = now + } + task.FailReason = taskInfo.Reason + task.PrivateData.ResultURL = taskInfo.Url + } + if taskInfo.Progress != "" { + task.Progress = taskInfo.Progress + } + } + } if insertErr := task.Insert(); insertErr != nil { common.SysError("insert task error: " + insertErr.Error()) } diff --git a/middleware/distributor.go b/middleware/distributor.go index d626941456c7..feaf3efb2ea1 100644 --- a/middleware/distributor.go +++ b/middleware/distributor.go @@ -81,8 +81,8 @@ func Distribute() func(c *gin.Context) { } var selectGroup string usingGroup := common.GetContextKeyString(c, constant.ContextKeyUsingGroup) - // check path is /pg/chat/completions - if strings.HasPrefix(c.Request.URL.Path, "/pg/chat/completions") { + // playground requests may override group in body + if strings.HasPrefix(c.Request.URL.Path, "/pg/") { playgroundRequest := &dto.PlayGroundRequest{} err = common.UnmarshalBodyReusable(c, playgroundRequest) if err != nil { diff --git a/relay/channel/task/sora/adaptor.go b/relay/channel/task/sora/adaptor.go index e9029aa20d46..5f1f7f4c88b4 100644 --- a/relay/channel/task/sora/adaptor.go +++ b/relay/channel/task/sora/adaptor.go @@ -21,6 +21,7 @@ import ( "github.com/gin-gonic/gin" "github.com/pkg/errors" + "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) @@ -44,6 +45,7 @@ type responseTask struct { Object string `json:"object"` Model string `json:"model"` Status string `json:"status"` + URL string `json:"url,omitempty"` Progress int `json:"progress"` CreatedAt int64 `json:"created_at"` CompletedAt int64 `json:"completed_at,omitempty"` @@ -68,6 +70,116 @@ type TaskAdaptor struct { baseURL string } +func stringifyBodyValue(value any) string { + if value == nil { + return "" + } + switch v := value.(type) { + case string: + return strings.TrimSpace(v) + default: + return strings.TrimSpace(fmt.Sprint(v)) + } +} + +func normalizeGrokVideoQuality(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "720p": + return "high" + case "480p": + return "standard" + case "high", "standard": + return strings.ToLower(strings.TrimSpace(value)) + default: + return strings.TrimSpace(value) + } +} + +func resolutionNameFromQuality(value string) string { + switch normalizeGrokVideoQuality(value) { + case "high": + return "720p" + case "standard": + return "480p" + default: + return "" + } +} + +func qualityFromResolutionName(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "720p": + return "high" + case "480p": + return "standard" + default: + return "" + } +} + +func normalizeGrokVideoRequest(bodyMap map[string]interface{}, upstreamModel string) { + if upstreamModel != "grok-imagine-1.0-video" { + return + } + + quality := normalizeGrokVideoQuality(stringifyBodyValue(bodyMap["quality"])) + resolutionName := stringifyBodyValue(bodyMap["resolution_name"]) + preset := stringifyBodyValue(bodyMap["preset"]) + + if videoConfig, ok := bodyMap["video_config"].(map[string]interface{}); ok { + if resolutionName == "" { + resolutionName = stringifyBodyValue(videoConfig["resolution_name"]) + } + if preset == "" { + preset = stringifyBodyValue(videoConfig["preset"]) + } + } + + if quality == "" { + quality = qualityFromResolutionName(resolutionName) + } + if resolutionName == "" { + resolutionName = resolutionNameFromQuality(quality) + } + + if quality != "" { + bodyMap["quality"] = quality + } + if resolutionName != "" { + bodyMap["resolution_name"] = resolutionName + } + if preset != "" { + bodyMap["preset"] = preset + } + if resolutionName != "" || preset != "" { + videoConfig := map[string]interface{}{} + if resolutionName != "" { + videoConfig["resolution_name"] = resolutionName + } + if preset != "" { + videoConfig["preset"] = preset + } + bodyMap["video_config"] = videoConfig + } +} + +func extractVideoURL(respBody []byte) string { + for _, path := range []string{ + "url", + "video_url", + "metadata.url", + "data.url", + "data.video_url", + "output.video_url", + "task_result.videos.0.url", + } { + if url := strings.TrimSpace(gjson.GetBytes(respBody, path).String()); url != "" { + return url + } + } + return "" +} + func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) { a.ChannelType = info.ChannelType a.baseURL = info.ChannelBaseUrl @@ -158,6 +270,7 @@ func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayIn var bodyMap map[string]interface{} if err := common.Unmarshal(cachedBody, &bodyMap); err == nil { bodyMap["model"] = info.UpstreamModelName + normalizeGrokVideoRequest(bodyMap, info.UpstreamModelName) if newBody, err := common.Marshal(bodyMap); err == nil { return bytes.NewReader(newBody), nil } @@ -248,6 +361,9 @@ func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *rela taskErr = service.TaskErrorWrapper(fmt.Errorf("task_id is empty"), "invalid_response", http.StatusInternalServerError) return } + if dResp.URL == "" { + dResp.URL = extractVideoURL(responseBody) + } // 使用公开 task_xxxx ID 返回给客户端 dResp.ID = info.PublicTaskID @@ -292,6 +408,9 @@ func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, e if err := common.Unmarshal(respBody, &resTask); err != nil { return nil, errors.Wrap(err, "unmarshal task result failed") } + if resTask.URL == "" { + resTask.URL = extractVideoURL(respBody) + } taskResult := relaycommon.TaskInfo{ Code: 0, @@ -304,6 +423,7 @@ func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, e taskResult.Status = model.TaskStatusInProgress case "completed": taskResult.Status = model.TaskStatusSuccess + taskResult.Url = resTask.URL // Url intentionally left empty — the caller constructs the proxy URL using the public task ID case "failed", "cancelled": taskResult.Status = model.TaskStatusFailure @@ -327,5 +447,10 @@ func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) { if data, err = sjson.SetBytes(data, "id", task.TaskID); err != nil { return nil, errors.Wrap(err, "set id failed") } + if gjson.GetBytes(data, "task_id").Exists() { + if data, err = sjson.SetBytes(data, "task_id", task.TaskID); err != nil { + return nil, errors.Wrap(err, "set task_id failed") + } + } return data, nil } diff --git a/relay/channel/task/sora/adaptor_test.go b/relay/channel/task/sora/adaptor_test.go new file mode 100644 index 000000000000..73a1040f166c --- /dev/null +++ b/relay/channel/task/sora/adaptor_test.go @@ -0,0 +1,47 @@ +package sora + +import "testing" + +func TestNormalizeGrokVideoRequestAddsResolutionAliases(t *testing.T) { + body := map[string]interface{}{ + "model": "grok-imagine-1.0-video", + "quality": "high", + "preset": "fun", + } + + normalizeGrokVideoRequest(body, "grok-imagine-1.0-video") + + if got := body["quality"]; got != "high" { + t.Fatalf("expected quality to stay high, got %#v", got) + } + if got := body["resolution_name"]; got != "720p" { + t.Fatalf("expected resolution_name 720p, got %#v", got) + } + + videoConfig, ok := body["video_config"].(map[string]interface{}) + if !ok { + t.Fatalf("expected video_config map, got %#v", body["video_config"]) + } + if got := videoConfig["resolution_name"]; got != "720p" { + t.Fatalf("expected video_config.resolution_name 720p, got %#v", got) + } + if got := videoConfig["preset"]; got != "fun" { + t.Fatalf("expected video_config.preset fun, got %#v", got) + } +} + +func TestNormalizeGrokVideoRequestBackfillsQualityFromResolutionName(t *testing.T) { + body := map[string]interface{}{ + "model": "grok-imagine-1.0-video", + "resolution_name": "720p", + } + + normalizeGrokVideoRequest(body, "grok-imagine-1.0-video") + + if got := body["quality"]; got != "high" { + t.Fatalf("expected quality high, got %#v", got) + } + if got := body["resolution_name"]; got != "720p" { + t.Fatalf("expected resolution_name 720p, got %#v", got) + } +} diff --git a/relay/channel/xai/adaptor.go b/relay/channel/xai/adaptor.go index e172bccf324a..8a4e48e86a1d 100644 --- a/relay/channel/xai/adaptor.go +++ b/relay/channel/xai/adaptor.go @@ -42,6 +42,7 @@ func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInf Model: request.Model, Prompt: request.Prompt, N: int(lo.FromPtrOr(request.N, uint(1))), + Image: request.Image, ResponseFormat: request.ResponseFormat, } return xaiRequest, nil diff --git a/relay/channel/xai/adaptor_test.go b/relay/channel/xai/adaptor_test.go new file mode 100644 index 000000000000..2ada3ec48a72 --- /dev/null +++ b/relay/channel/xai/adaptor_test.go @@ -0,0 +1,60 @@ +package xai + +import ( + "encoding/json" + "testing" + + "github.com/QuantumNous/new-api/dto" + "github.com/samber/lo" +) + +func TestConvertImageRequestPreservesEditImage(t *testing.T) { + adaptor := &Adaptor{} + image := json.RawMessage(`{"url":"https://example.com/source.png"}`) + + converted, err := adaptor.ConvertImageRequest(nil, nil, dto.ImageRequest{ + Model: "grok-imagine-1.0-edit", + Prompt: "make it watercolor", + N: lo.ToPtr(uint(2)), + Image: image, + ResponseFormat: "url", + }) + if err != nil { + t.Fatalf("ConvertImageRequest returned error: %v", err) + } + + xaiReq, ok := converted.(ImageRequest) + if !ok { + t.Fatalf("expected xai.ImageRequest, got %T", converted) + } + if xaiReq.Model != "grok-imagine-1.0-edit" { + t.Fatalf("unexpected model: %s", xaiReq.Model) + } + if xaiReq.N != 2 { + t.Fatalf("unexpected n: %d", xaiReq.N) + } + if xaiReq.ResponseFormat != "url" { + t.Fatalf("unexpected response_format: %s", xaiReq.ResponseFormat) + } + gotImage, ok := xaiReq.Image.(json.RawMessage) + if !ok { + t.Fatalf("expected json.RawMessage image, got %T", xaiReq.Image) + } + if string(gotImage) != string(image) { + t.Fatalf("unexpected image payload: %s", string(gotImage)) + } +} + +func TestModelListIncludesGrokImagineOnePointZeroVariants(t *testing.T) { + expected := []string{ + "grok-imagine-1.0", + "grok-imagine-1.0-fast", + "grok-imagine-1.0-edit", + } + + for _, model := range expected { + if !lo.Contains(ModelList, model) { + t.Fatalf("model list missing %s", model) + } + } +} diff --git a/relay/channel/xai/constants.go b/relay/channel/xai/constants.go index c20532d4ce78..5241b375d1f4 100644 --- a/relay/channel/xai/constants.go +++ b/relay/channel/xai/constants.go @@ -22,6 +22,9 @@ var ModelList = []string{ // grok-3-mini reasoning effort variants "grok-3-mini-high", "grok-3-mini-low", // image generation models + "grok-imagine-1.0", + "grok-imagine-1.0-fast", + "grok-imagine-1.0-edit", "grok-imagine-image-pro", "grok-imagine-image", "grok-2-image-1212", diff --git a/relay/channel/xai/dto.go b/relay/channel/xai/dto.go index 371d62a43360..617d32f290ce 100644 --- a/relay/channel/xai/dto.go +++ b/relay/channel/xai/dto.go @@ -18,6 +18,7 @@ type ImageRequest struct { Model string `json:"model"` Prompt string `json:"prompt" binding:"required"` N int `json:"n,omitempty"` + Image any `json:"image,omitempty"` // Size string `json:"size,omitempty"` // Quality string `json:"quality,omitempty"` ResponseFormat string `json:"response_format,omitempty"` diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index ef1411af156d..a3c994bb6779 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -672,6 +672,9 @@ type TaskSubmitReq struct { Size string `json:"size,omitempty"` Duration int `json:"duration,omitempty"` Seconds string `json:"seconds,omitempty"` + Quality string `json:"quality,omitempty"` + ResolutionName string `json:"resolution_name,omitempty"` + Preset string `json:"preset,omitempty"` InputReference string `json:"input_reference,omitempty"` Metadata map[string]interface{} `json:"metadata,omitempty"` } diff --git a/relay/common/relay_utils.go b/relay/common/relay_utils.go index 3cbb18c22c2f..993b6a3b6422 100644 --- a/relay/common/relay_utils.go +++ b/relay/common/relay_utils.go @@ -190,6 +190,10 @@ func isKnownTaskField(field string) bool { "images": true, "size": true, "duration": true, + "seconds": true, + "quality": true, + "resolution_name": true, + "preset": true, "input_reference": true, // Sora 特有字段 } return knownFields[field] diff --git a/relay/constant/relay_mode.go b/relay/constant/relay_mode.go index 256715679213..c34ba687a037 100644 --- a/relay/constant/relay_mode.go +++ b/relay/constant/relay_mode.go @@ -66,9 +66,9 @@ func Path2RelayMode(path string) int { relayMode = RelayModeEmbeddings } else if strings.HasPrefix(path, "/v1/moderations") { relayMode = RelayModeModerations - } else if strings.HasPrefix(path, "/v1/images/generations") { + } else if strings.HasPrefix(path, "/v1/images/generations") || strings.HasPrefix(path, "/pg/images/generations") { relayMode = RelayModeImagesGenerations - } else if strings.HasPrefix(path, "/v1/images/edits") { + } else if strings.HasPrefix(path, "/v1/images/edits") || strings.HasPrefix(path, "/pg/images/edits") { relayMode = RelayModeImagesEdits } else if strings.HasPrefix(path, "/v1/edits") { relayMode = RelayModeEdits diff --git a/relay/constant/relay_mode_test.go b/relay/constant/relay_mode_test.go new file mode 100644 index 000000000000..706b019e8def --- /dev/null +++ b/relay/constant/relay_mode_test.go @@ -0,0 +1,20 @@ +package constant + +import "testing" + +func TestPath2RelayModeSupportsPlaygroundImageRoutes(t *testing.T) { + tests := []struct { + path string + want int + }{ + {path: "/pg/images/generations", want: RelayModeImagesGenerations}, + {path: "/pg/images/edits", want: RelayModeImagesEdits}, + {path: "/pg/chat/completions", want: RelayModeChatCompletions}, + } + + for _, tt := range tests { + if got := Path2RelayMode(tt.path); got != tt.want { + t.Fatalf("Path2RelayMode(%q) = %d, want %d", tt.path, got, tt.want) + } + } +} diff --git a/router/relay-router.go b/router/relay-router.go index 830288077e6e..9dce007f92b4 100644 --- a/router/relay-router.go +++ b/router/relay-router.go @@ -65,6 +65,8 @@ func SetRelayRouter(router *gin.Engine) { playgroundRouter.Use(middleware.UserAuth(), middleware.Distribute()) { playgroundRouter.POST("/chat/completions", controller.Playground) + playgroundRouter.POST("/images/generations", controller.PlaygroundImageGenerations) + playgroundRouter.POST("/images/edits", controller.PlaygroundImageEdits) playgroundRouter.POST("/video/generations", controller.PlaygroundVideoSubmit) playgroundRouter.GET("/video/generations/:task_id", controller.PlaygroundVideoFetch) } diff --git a/web/src/components/playground/SettingsPanel.jsx b/web/src/components/playground/SettingsPanel.jsx index 6bbcbc5fc0c4..1ece90f888f1 100644 --- a/web/src/components/playground/SettingsPanel.jsx +++ b/web/src/components/playground/SettingsPanel.jsx @@ -49,6 +49,7 @@ const SettingsPanel = ({ const { t } = useTranslation(); const isVideoModel = typeof inputs.model === 'string' && inputs.model.includes('video'); + const isGrokImagineVideoModel = inputs.model === 'grok-imagine-1.0-video'; const videoSizeOptions = [ { label: '1280x720', value: '1280x720' }, { label: '720x1280', value: '720x1280' }, @@ -60,9 +61,15 @@ const SettingsPanel = ({ label: `${v}s`, value: String(v), })); + const videoPresetOptions = [ + { label: 'Normal', value: 'normal' }, + { label: 'Fun', value: 'fun' }, + { label: 'Spicy', value: 'spicy' }, + { label: 'Custom', value: 'custom' }, + ]; const videoQualityOptions = [ - { label: 'standard', value: 'standard' }, - { label: 'high', value: 'high' }, + { label: '480p', value: '480p' }, + { label: '720p', value: '720p' }, ]; const currentConfig = { @@ -245,6 +252,20 @@ const SettingsPanel = ({ disabled={customRequestMode} /> + {isGrokImagineVideoModel && ( +
+ + {t('风格预设')} + + onInputChange('videoQuality', value)} disabled={customRequestMode} /> diff --git a/web/src/components/table/models/modals/EditModelModal.jsx b/web/src/components/table/models/modals/EditModelModal.jsx index 12ca48ec1a62..01e2d173b372 100644 --- a/web/src/components/table/models/modals/EditModelModal.jsx +++ b/web/src/components/table/models/modals/EditModelModal.jsx @@ -50,6 +50,7 @@ const ENDPOINT_TEMPLATE = { gemini: { path: '/v1beta/models/{model}:generateContent', method: 'POST' }, 'jina-rerank': { path: '/v1/rerank', method: 'POST' }, 'image-generation': { path: '/v1/images/generations', method: 'POST' }, + 'image-edit': { path: '/v1/images/edits', method: 'POST' }, }; const nameRuleOptions = [ diff --git a/web/src/components/table/models/modals/EditPrefillGroupModal.jsx b/web/src/components/table/models/modals/EditPrefillGroupModal.jsx index bb7af45cdb82..a90c3f30f516 100644 --- a/web/src/components/table/models/modals/EditPrefillGroupModal.jsx +++ b/web/src/components/table/models/modals/EditPrefillGroupModal.jsx @@ -48,6 +48,7 @@ const ENDPOINT_TEMPLATE = { gemini: { path: '/v1beta/models/{model}:generateContent', method: 'POST' }, 'jina-rerank': { path: '/v1/rerank', method: 'POST' }, 'image-generation': { path: '/v1/images/generations', method: 'POST' }, + 'image-edit': { path: '/v1/images/edits', method: 'POST' }, }; const EditPrefillGroupModal = ({ diff --git a/web/src/constants/playground.constants.js b/web/src/constants/playground.constants.js index 9751eb67224f..1361919fcedb 100644 --- a/web/src/constants/playground.constants.js +++ b/web/src/constants/playground.constants.js @@ -76,6 +76,8 @@ export const DEBUG_TABS = { // ========== API 相关常量 ========== export const API_ENDPOINTS = { CHAT_COMPLETIONS: '/pg/chat/completions', + IMAGE_GENERATIONS: '/pg/images/generations', + IMAGE_EDITS: '/pg/images/edits', VIDEO_GENERATIONS: '/pg/video/generations', USER_MODELS: '/api/user/models', USER_GROUPS: '/api/user/self/groups', @@ -97,7 +99,8 @@ export const DEFAULT_CONFIG = { imageUrls: [''], videoSize: '1280x720', videoSeconds: '10', - videoQuality: 'standard', + videoQuality: '480p', + videoPreset: 'normal', }, parameterEnabled: { temperature: true, diff --git a/web/src/helpers/api.js b/web/src/helpers/api.js index dcb8b5334e79..2279debddb07 100644 --- a/web/src/helpers/api.js +++ b/web/src/helpers/api.js @@ -115,6 +115,11 @@ export const buildApiPayload = ( inputs, parameterEnabled, ) => { + const grokImagineImageModels = new Set([ + 'grok-imagine-1.0', + 'grok-imagine-1.0-fast', + 'grok-imagine-1.0-edit', + ]); const processedMessages = messages .filter(isValidMessage) .map(formatMessageForAPI) @@ -157,6 +162,11 @@ export const buildApiPayload = ( const isVideoModel = typeof inputs.model === 'string' && inputs.model.includes('video'); + const isGrokImagineImageModel = grokImagineImageModels.has(inputs.model); + const isGrokImagineVideoModel = inputs.model === 'grok-imagine-1.0-video'; + if (isGrokImagineImageModel) { + payload.stream = false; + } if (isVideoModel) { payload.stream = false; if (inputs.videoSize) { @@ -166,7 +176,32 @@ export const buildApiPayload = ( payload.seconds = String(inputs.videoSeconds); } if (inputs.videoQuality) { - payload.quality = inputs.videoQuality; + const resolutionName = + inputs.videoQuality === 'high' + ? '720p' + : inputs.videoQuality === 'standard' + ? '480p' + : inputs.videoQuality; + payload.quality = + resolutionName === '720p' + ? 'high' + : resolutionName === '480p' + ? 'standard' + : resolutionName; + if (isGrokImagineVideoModel && resolutionName) { + payload.resolution_name = resolutionName; + } + } + if (isGrokImagineVideoModel && inputs.videoPreset) { + payload.preset = inputs.videoPreset; + } + if (isGrokImagineVideoModel && (payload.resolution_name || payload.preset)) { + payload.video_config = { + ...(payload.resolution_name + ? { resolution_name: payload.resolution_name } + : {}), + ...(payload.preset ? { preset: payload.preset } : {}), + }; } } diff --git a/web/src/hooks/playground/useApiRequest.jsx b/web/src/hooks/playground/useApiRequest.jsx index 405eb3234207..d02639bc3c04 100644 --- a/web/src/hooks/playground/useApiRequest.jsx +++ b/web/src/hooks/playground/useApiRequest.jsx @@ -32,6 +32,12 @@ import { processIncompleteThinkTags, } from '../../helpers'; +const GROK_IMAGE_GENERATION_MODELS = new Set([ + 'grok-imagine-1.0', + 'grok-imagine-1.0-fast', +]); +const GROK_IMAGE_EDIT_MODELS = new Set(['grok-imagine-1.0-edit']); + export const useApiRequest = ( setMessage, setDebugData, @@ -40,11 +46,30 @@ export const useApiRequest = ( saveMessages, ) => { const { t } = useTranslation(); + + const isGrokImagineImageModel = useCallback((model) => { + return ( + GROK_IMAGE_GENERATION_MODELS.has(model) || GROK_IMAGE_EDIT_MODELS.has(model) + ); + }, []); + + const isGrokImagineImageEditModel = useCallback((model) => { + return GROK_IMAGE_EDIT_MODELS.has(model); + }, []); + const isVideoGenerationPayload = useCallback((payload) => { const model = payload?.model; return typeof model === 'string' && model.includes('video'); }, []); + const isImageGenerationPayload = useCallback( + (payload) => { + const model = payload?.model; + return typeof model === 'string' && isGrokImagineImageModel(model); + }, + [isGrokImagineImageModel], + ); + const getTextFromMessageContent = useCallback((content) => { if (typeof content === 'string') { return content; @@ -74,6 +99,26 @@ export const useApiRequest = ( return imageURL?.url || ''; }, []); + const normalizeVideoQuality = useCallback((quality) => { + if (quality === '720p') { + return 'high'; + } + if (quality === '480p') { + return 'standard'; + } + return quality; + }, []); + + const formatVideoQuality = useCallback((quality) => { + if (quality === 'high' || quality === '720p') { + return '720p'; + } + if (quality === 'standard' || quality === '480p') { + return '480p'; + } + return quality || ''; + }, []); + const buildVideoRequestPayload = useCallback( (payload) => { const messages = Array.isArray(payload?.messages) ? payload.messages : []; @@ -82,21 +127,112 @@ export const useApiRequest = ( .find((m) => m?.role === 'user'); const prompt = getTextFromMessageContent(lastUserMessage?.content); const image = getImageFromMessageContent(lastUserMessage?.content); - - return { + const size = payload?.size || payload?.videoSize; + const seconds = payload?.seconds || payload?.videoSeconds; + const quality = payload?.quality || payload?.videoQuality; + const preset = payload?.preset || payload?.videoPreset; + const isGrokImagineVideoModel = payload?.model === 'grok-imagine-1.0-video'; + const resolutionName = + payload?.resolution_name || + (isGrokImagineVideoModel ? formatVideoQuality(quality) : ''); + + const requestPayload = { model: payload.model, prompt, - seconds: payload.seconds, - size: payload.size, - quality: payload.quality, + seconds, + size, + quality: normalizeVideoQuality(quality), + preset, ...(image ? { image } : {}), }; + + if (isGrokImagineVideoModel && resolutionName) { + requestPayload.resolution_name = resolutionName; + requestPayload.video_config = { + resolution_name: resolutionName, + ...(preset ? { preset } : {}), + }; + } + + return requestPayload; }, - [getImageFromMessageContent, getTextFromMessageContent], + [ + formatVideoQuality, + getImageFromMessageContent, + getTextFromMessageContent, + normalizeVideoQuality, + ], ); + const buildImageRequestPayload = useCallback( + (payload) => { + const messages = Array.isArray(payload?.messages) ? payload.messages : []; + const lastUserMessage = [...messages] + .reverse() + .find((m) => m?.role === 'user'); + const prompt = getTextFromMessageContent(lastUserMessage?.content); + const image = getImageFromMessageContent(lastUserMessage?.content); + const resolvedPrompt = + prompt || + (isGrokImagineImageEditModel(payload.model) + ? 'Edit the provided media.' + : ''); + const requestPayload = { + model: payload.model, + group: payload.group, + prompt: resolvedPrompt, + n: 1, + response_format: 'url', + }; + + if (isGrokImagineImageEditModel(payload.model) && image) { + requestPayload.image = { url: image }; + } + + return requestPayload; + }, + [ + getImageFromMessageContent, + getTextFromMessageContent, + isGrokImagineImageEditModel, + ], + ); + + const extractVideoUrl = useCallback((payload) => { + if (!payload || typeof payload !== 'object') { + return ''; + } + + const candidates = [ + payload.url, + payload.video_url, + payload.result_url, + payload.metadata?.url, + payload.data?.url, + payload.data?.video_url, + payload.data?.result_url, + payload.data?.metadata?.url, + ]; + + const matched = candidates.find( + (item) => typeof item === 'string' && item.trim() !== '', + ); + + return matched?.trim() || ''; + }, []); + const resolveEndpointAndPayload = useCallback( (payload) => { + if (isImageGenerationPayload(payload)) { + const requestPayload = buildImageRequestPayload(payload); + return { + endpoint: isGrokImagineImageEditModel(payload?.model) + ? API_ENDPOINTS.IMAGE_EDITS + : API_ENDPOINTS.IMAGE_GENERATIONS, + requestPayload, + forceNonStream: true, + }; + } if (isVideoGenerationPayload(payload)) { return { endpoint: API_ENDPOINTS.VIDEO_GENERATIONS, @@ -110,7 +246,13 @@ export const useApiRequest = ( forceNonStream: false, }; }, - [buildVideoRequestPayload, isVideoGenerationPayload], + [ + buildImageRequestPayload, + buildVideoRequestPayload, + isGrokImagineImageEditModel, + isImageGenerationPayload, + isVideoGenerationPayload, + ], ); // 处理消息自动关闭逻辑的公共函数 @@ -306,12 +448,23 @@ export const useApiRequest = ( data.object === 'video' || data.task_id ) { + const videoUrl = extractVideoUrl(data); + const requestedQuality = formatVideoQuality(requestPayload.quality); + const upstreamQuality = formatVideoQuality(data.quality); const summary = [ `${t('视频任务已创建')}`, `task_id: ${data.task_id || data.id || '-'}`, `status: ${data.status || '-'}`, `seconds: ${data.seconds || requestPayload.seconds || '-'}`, `size: ${data.size || requestPayload.size || '-'}`, + `quality: ${requestedQuality || upstreamQuality || '-'}`, + ...(upstreamQuality && upstreamQuality !== requestedQuality + ? [`upstream_quality: ${upstreamQuality}`] + : []), + ...(requestPayload.preset ? [`preset: ${requestPayload.preset}`] : []), + ...(videoUrl + ? [`url: \`${videoUrl}\``, `[Open Video](${videoUrl})`] + : []), ].join('\n'); setMessage((prevMessage) => { const newMessages = [...prevMessage]; @@ -333,6 +486,52 @@ export const useApiRequest = ( return; } + if ( + endpoint === API_ENDPOINTS.IMAGE_GENERATIONS || + endpoint === API_ENDPOINTS.IMAGE_EDITS || + Array.isArray(data.data) + ) { + const imageUrls = (Array.isArray(data.data) ? data.data : []) + .map((item) => item?.url) + .filter((item) => typeof item === 'string' && item.trim() !== ''); + + const summaryLines = [ + endpoint === API_ENDPOINTS.IMAGE_EDITS + ? t('图片编辑已完成') + : t('图片任务已完成'), + `model: ${requestPayload.model || payload?.model || '-'}`, + `count: ${imageUrls.length || data.data?.length || 0}`, + ]; + + imageUrls.forEach((url, index) => { + summaryLines.push(`image_${index + 1}: [Open Image ${index + 1}](${url})`); + summaryLines.push(`![image_${index + 1}](${url})`); + }); + + if (imageUrls.length === 0) { + summaryLines.push(t('未在响应中解析到图片链接')); + } + + setMessage((prevMessage) => { + const newMessages = [...prevMessage]; + const lastMessage = newMessages[newMessages.length - 1]; + if (lastMessage?.status === MESSAGE_STATUS.LOADING) { + const autoCollapseState = applyAutoCollapseLogic( + lastMessage, + true, + ); + newMessages[newMessages.length - 1] = { + ...lastMessage, + content: summaryLines.join('\n\n'), + status: MESSAGE_STATUS.COMPLETE, + ...autoCollapseState, + }; + } + return newMessages; + }); + return; + } + if (data.choices?.[0]) { const choice = data.choices[0]; let content = choice.message?.content || ''; @@ -396,6 +595,8 @@ export const useApiRequest = ( setActiveDebugTab, setMessage, t, + extractVideoUrl, + formatVideoQuality, applyAutoCollapseLogic, ], );